jh163888/xrdp
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

fixes for rle bitmap compress limits and raw bitmaps

Browse Source
This commit is contained in:
Jay Sorg 2016-12-14 23:55:13 -08:00
parent 4462d22bf1
commit 492116d535
3 changed files with 159 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

86
libxrdp/xrdp_bitmap32_compress.c
View File

@ -107,7 +107,7 @@ fsplit3(char *in_data, int start_line, int width, int e,
} }
start_line--; start_line--;
cy++; cy++;
if (out_index > 64 * 64) if (out_index + width + e > 64 * 64)
{ {
break; break;
} }
@ -195,7 +195,7 @@ fsplit4(char *in_data, int start_line, int width, int e,
} }
start_line--; start_line--;
cy++; cy++;
if (out_index > 64 * 64) if (out_index + width + e > 64 * 64)
{ {
break; break;
} }
@ -422,6 +422,7 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
char *sr_data; char *sr_data;
char *sg_data; char *sg_data;
char *sb_data; char *sb_data;
char *hold_p;
int a_bytes; int a_bytes;
int r_bytes; int r_bytes;
int g_bytes; int g_bytes;
@ -449,6 +450,7 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
r_data = a_data + max_bytes; r_data = a_data + max_bytes;
g_data = r_data + max_bytes; g_data = r_data + max_bytes;
b_data = g_data + max_bytes; b_data = g_data + max_bytes;
hold_p = s->p;
if (header & FLAGS_NOALPHA) if (header & FLAGS_NOALPHA)
{ {
@ -459,35 +461,44 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
fdelta(sr_data, r_data, cx, cy); fdelta(sr_data, r_data, cx, cy);
fdelta(sg_data, g_data, cx, cy); fdelta(sg_data, g_data, cx, cy);
fdelta(sb_data, b_data, cx, cy); fdelta(sb_data, b_data, cx, cy);
while (cy > 0)
{
s->p = hold_p;
out_uint8(s, header); out_uint8(s, header);
r_bytes = fpack(r_data, cx, cy, s); r_bytes = fpack(r_data, cx, cy, s);
g_bytes = fpack(g_data, cx, cy, s); g_bytes = fpack(g_data, cx, cy, s);
b_bytes = fpack(b_data, cx, cy, s); b_bytes = fpack(b_data, cx, cy, s);
total_bytes = r_bytes + g_bytes + b_bytes;
if (1 + total_bytes > byte_limit)
{
/* failed */
LLOGLN(0, ("xrdp_bitmap32_compress: too big, rgb "
"bytes %d %d %d total_bytes %d cx %d cy %d "
"byte_limit %d", r_bytes, g_bytes, b_bytes,
total_bytes, cx, cy, byte_limit));
return 0;
}
max_bytes = cx * cy * 3; max_bytes = cx * cy * 3;
total_bytes = r_bytes + g_bytes + b_bytes;
if (total_bytes > max_bytes) if (total_bytes > max_bytes)
{ {
/* raw is better */ if (2 + max_bytes <= byte_limit)
LLOGLN(10, ("xrdp_bitmap32_compress: too big, rgb " {
"bytes %d %d %d total_bytes %d cx %d cy %d " s->p = hold_p;
"max_bytes %d", r_bytes, g_bytes, b_bytes,
total_bytes, cx, cy, max_bytes));
init_stream(s, 0);
foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data); foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data);
break;
}
}
if (1 + total_bytes <= byte_limit)
{
break;
}
cy--;
} }
} }
else else
{ {
while (cy > 0)
{
max_bytes = cx * cy * 3;
if (2 + max_bytes <= byte_limit)
{
s->p = hold_p;
foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data); foutraw3(s, cx * cy, FLAGS_NOALPHA, sr_data, sg_data, sb_data);
break;
}
cy--;
}
} }
} }
else else
@ -500,6 +511,9 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
fdelta(sr_data, r_data, cx, cy); fdelta(sr_data, r_data, cx, cy);
fdelta(sg_data, g_data, cx, cy); fdelta(sg_data, g_data, cx, cy);
fdelta(sb_data, b_data, cx, cy); fdelta(sb_data, b_data, cx, cy);
while (cy > 0)
{
s->p = hold_p;
out_uint8(s, header); out_uint8(s, header);
a_bytes = fpack(a_data, cx, cy, s); a_bytes = fpack(a_data, cx, cy, s);
r_bytes = fpack(r_data, cx, cy, s); r_bytes = fpack(r_data, cx, cy, s);
@ -507,29 +521,35 @@ xrdp_bitmap32_compress(char *in_data, int width, int height,
b_bytes = fpack(b_data, cx, cy, s); b_bytes = fpack(b_data, cx, cy, s);
max_bytes = cx * cy * 4; max_bytes = cx * cy * 4;
total_bytes = a_bytes + r_bytes + g_bytes + b_bytes; total_bytes = a_bytes + r_bytes + g_bytes + b_bytes;
if (1 + total_bytes > byte_limit)
{
/* failed */
LLOGLN(0, ("xrdp_bitmap32_compress: too big, argb "
"bytes %d %d %d %d total_bytes %d cx %d cy %d "
"byte_limit %d", a_bytes, r_bytes, g_bytes, b_bytes,
total_bytes, cx, cy, byte_limit));
return 0;
}
if (total_bytes > max_bytes) if (total_bytes > max_bytes)
{ {
/* raw is better */ if (2 + max_bytes <= byte_limit)
LLOGLN(10, ("xrdp_bitmap32_compress: too big, argb " {
"bytes %d %d %d %d total_bytes %d cx %d cy %d " s->p = hold_p;
"max_bytes %d", a_bytes, r_bytes, g_bytes, b_bytes,
total_bytes, cx, cy, max_bytes));
init_stream(s, 0);
foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data); foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data);
break;
}
}
if (1 + total_bytes <= byte_limit)
{
break;
}
cy--;
} }
} }
else else
{ {
while (cy > 0)
{
max_bytes = cx * cy * 4;
if (2 + max_bytes <= byte_limit)
{
s->p = hold_p;
foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data); foutraw4(s, cx * cy, 0, sa_data, sr_data, sg_data, sb_data);
break;
}
cy--;
}
} }
} }
return cy; return cy;

8
libxrdp/xrdp_bitmap_compress.c
View File

@ -22,6 +22,8 @@
#include "libxrdp.h" #include "libxrdp.h"
#define BC_MAX_BYTES (16 * 1024)
/*****************************************************************************/ /*****************************************************************************/
#define IN_PIXEL8(in_ptr, in_x, in_y, in_w, in_last_pixel, in_pixel); \ #define IN_PIXEL8(in_ptr, in_x, in_y, in_w, in_last_pixel, in_pixel); \
do { \ do { \
@ -695,7 +697,7 @@ xrdp_bitmap_compress(char *in_data, int width, int height,
out_count = end; out_count = end;
line = in_data + width * start_line; line = in_data + width * start_line;
while (start_line >= 0 && out_count < 32768) while (start_line >= 0 && out_count <= BC_MAX_BYTES)
{ {
i = (s->p - s->data) + count; i = (s->p - s->data) + count;
@ -987,7 +989,7 @@ xrdp_bitmap_compress(char *in_data, int width, int height,
out_count = end * 2; out_count = end * 2;
line = in_data + width * start_line * 2; line = in_data + width * start_line * 2;
while (start_line >= 0 && out_count < 32768) while (start_line >= 0 && out_count <= BC_MAX_BYTES)
{ {
i = (s->p - s->data) + count * 2; i = (s->p - s->data) + count * 2;
@ -1279,7 +1281,7 @@ xrdp_bitmap_compress(char *in_data, int width, int height,
out_count = end * 3; out_count = end * 3;
line = in_data + width * start_line * 4; line = in_data + width * start_line * 4;
while (start_line >= 0 && out_count < 32768) while (start_line >= 0 && out_count <= BC_MAX_BYTES)
{ {
i = (s->p - s->data) + count * 3; i = (s->p - s->data) + count * 3;

119
libxrdp/xrdp_orders.c
View File

@ -2235,6 +2235,11 @@ xrdp_orders_send_raw_bitmap(struct xrdp_orders *self,
Bpp = (bpp + 7) / 8; Bpp = (bpp + 7) / 8;
bufsize = (width + e) * height * Bpp; bufsize = (width + e) * height * Bpp;
while (bufsize + 16 > MAX_ORDERS_SIZE)
{
height--;
bufsize = (width + e) * height * Bpp;
}
if (xrdp_orders_check(self, bufsize + 16) != 0) if (xrdp_orders_check(self, bufsize + 16) != 0)
{ {
return 1; return 1;
@ -2254,33 +2259,58 @@ xrdp_orders_send_raw_bitmap(struct xrdp_orders *self,
out_uint16_le(self->out_s, bufsize); out_uint16_le(self->out_s, bufsize);
out_uint16_le(self->out_s, cache_idx); out_uint16_le(self->out_s, cache_idx);
if (Bpp == 4)
{
for (i = height - 1; i >= 0; i--) for (i = height - 1; i >= 0; i--)
{ {
for (j = 0; j < width; j++) for (j = 0; j < width; j++)
{ {
if (Bpp == 3) pixel = GETPIXEL32(data, j, i, width);
out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8);
out_uint8(self->out_s, pixel >> 16);
out_uint8(self->out_s, pixel >> 24);
}
out_uint8s(self->out_s, Bpp * e);
}
}
else if (Bpp == 3)
{
for (i = height - 1; i >= 0; i--)
{
for (j = 0; j < width; j++)
{ {
pixel = GETPIXEL32(data, j, i, width); pixel = GETPIXEL32(data, j, i, width);
out_uint8(self->out_s, pixel); out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8); out_uint8(self->out_s, pixel >> 8);
out_uint8(self->out_s, pixel >> 16); out_uint8(self->out_s, pixel >> 16);
} }
out_uint8s(self->out_s, Bpp * e);
}
}
else if (Bpp == 2) else if (Bpp == 2)
{
for (i = height - 1; i >= 0; i--)
{
for (j = 0; j < width; j++)
{ {
pixel = GETPIXEL16(data, j, i, width); pixel = GETPIXEL16(data, j, i, width);
out_uint8(self->out_s, pixel); out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8); out_uint8(self->out_s, pixel >> 8);
} }
out_uint8s(self->out_s, Bpp * e);
}
}
else if (Bpp == 1) else if (Bpp == 1)
{
for (i = height - 1; i >= 0; i--)
{
for (j = 0; j < width; j++)
{ {
pixel = GETPIXEL8(data, j, i, width); pixel = GETPIXEL8(data, j, i, width);
out_uint8(self->out_s, pixel); out_uint8(self->out_s, pixel);
} }
} out_uint8s(self->out_s, Bpp * e);
for (j = 0; j < e; j++)
{
out_uint8s(self->out_s, Bpp);
} }
} }
@ -2334,20 +2364,19 @@ xrdp_orders_send_bitmap(struct xrdp_orders *self,
if (bpp > 24) if (bpp > 24)
{ {
lines_sending = xrdp_bitmap32_compress(data, width, height, s, lines_sending = xrdp_bitmap32_compress(data, width, height, s,
bpp, 16384, bpp, MAX_ORDERS_SIZE,
i - 1, temp_s, e, 0x10); i - 1, temp_s, e, 0x10);
} }
else else
{ {
lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384, lines_sending = xrdp_bitmap_compress(data, width, height, s,
bpp, MAX_ORDERS_SIZE,
i - 1, temp_s, e); i - 1, temp_s, e);
} }
if (lines_sending != height) if (lines_sending != height)
{ {
g_writeln("error in xrdp_orders_send_bitmap, lines_sending(%d) != \ height = lines_sending;
height(%d)", lines_sending, height);
return 1;
} }
bufsize = (int)(s->p - p); bufsize = (int)(s->p - p);
@ -2458,6 +2487,7 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self,
int pixel = 0; int pixel = 0;
int e = 0; int e = 0;
g_writeln("xrdp_orders_send_raw_bitmap2:");
if (width > 64) if (width > 64)
{ {
g_writeln("error, width > 64"); g_writeln("error, width > 64");
@ -2479,6 +2509,11 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self,
Bpp = (bpp + 7) / 8; Bpp = (bpp + 7) / 8;
bufsize = (width + e) * height * Bpp; bufsize = (width + e) * height * Bpp;
while (bufsize + 14 > MAX_ORDERS_SIZE)
{
height--;
bufsize = (width + e) * height * Bpp;
}
if (xrdp_orders_check(self, bufsize + 14) != 0) if (xrdp_orders_check(self, bufsize + 14) != 0)
{ {
return 1; return 1;
@ -2499,7 +2534,22 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self,
i = cache_idx & 0xff; i = cache_idx & 0xff;
out_uint8(self->out_s, i); out_uint8(self->out_s, i);
if (1 && Bpp == 3) if (Bpp == 4)
{
for (i = height - 1; i >= 0; i--)
{
for (j = 0; j < width; j++)
{
pixel = GETPIXEL32(data, j, i, width);
out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8);
out_uint8(self->out_s, pixel >> 16);
out_uint8(self->out_s, pixel >> 24);
}
out_uint8s(self->out_s, Bpp * e);
}
}
else if (Bpp == 3)
{ {
for (i = height - 1; i >= 0; i--) for (i = height - 1; i >= 0; i--)
{ {
@ -2510,42 +2560,32 @@ xrdp_orders_send_raw_bitmap2(struct xrdp_orders *self,
out_uint8(self->out_s, pixel >> 8); out_uint8(self->out_s, pixel >> 8);
out_uint8(self->out_s, pixel >> 16); out_uint8(self->out_s, pixel >> 16);
} }
for (j = 0; j < e; j++) out_uint8s(self->out_s, Bpp * e);
{
out_uint8s(self->out_s, Bpp);
} }
} }
}
else
{
for (i = height - 1; i >= 0; i--)
{
for (j = 0; j < width; j++)
{
if (Bpp == 3)
{
pixel = GETPIXEL32(data, j, i, width);
out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8);
out_uint8(self->out_s, pixel >> 16);
}
else if (Bpp == 2) else if (Bpp == 2)
{
for (i = height - 1; i >= 0; i--)
{
for (j = 0; j < width; j++)
{ {
pixel = GETPIXEL16(data, j, i, width); pixel = GETPIXEL16(data, j, i, width);
out_uint8(self->out_s, pixel); out_uint8(self->out_s, pixel);
out_uint8(self->out_s, pixel >> 8); out_uint8(self->out_s, pixel >> 8);
} }
out_uint8s(self->out_s, Bpp * e);
}
}
else if (Bpp == 1) else if (Bpp == 1)
{
for (i = height - 1; i >= 0; i--)
{
for (j = 0; j < width; j++)
{ {
pixel = GETPIXEL8(data, j, i, width); pixel = GETPIXEL8(data, j, i, width);
out_uint8(self->out_s, pixel); out_uint8(self->out_s, pixel);
} }
} out_uint8s(self->out_s, Bpp * e);
for (j = 0; j < e; j++)
{
out_uint8s(self->out_s, Bpp);
}
} }
} }
@ -2599,20 +2639,19 @@ xrdp_orders_send_bitmap2(struct xrdp_orders *self,
if (bpp > 24) if (bpp > 24)
{ {
lines_sending = xrdp_bitmap32_compress(data, width, height, s, lines_sending = xrdp_bitmap32_compress(data, width, height, s,
bpp, 16384, bpp, MAX_ORDERS_SIZE,
i - 1, temp_s, e, 0x10); i - 1, temp_s, e, 0x10);
} }
else else
{ {
lines_sending = xrdp_bitmap_compress(data, width, height, s, bpp, 16384, lines_sending = xrdp_bitmap_compress(data, width, height, s,
bpp, MAX_ORDERS_SIZE,
i - 1, temp_s, e); i - 1, temp_s, e);
} }
if (lines_sending != height) if (lines_sending != height)
{ {
g_writeln("error in xrdp_orders_send_bitmap2, lines_sending(%d) != \ height = lines_sending;
height(%d)", lines_sending, height);
return 1;
} }
bufsize = (int)(s->p - p); bufsize = (int)(s->p - p);