work on codec mode

common/xrdp_client_info.h

@@ -122,6 +122,8 @@ struct xrdp_client_info
   int mcs_early_capability_flags;
 
   int max_fastpath_frag_bytes;
+  int capture_code;
+  int capture_format;
 
 };
 

librfxcodec

@@ -1 +1 @@
-Subproject commit 7b04ca9c54aeddbdaf4bf945cea81f4efb8847e7
+Subproject commit 766b06b5f914e9a0b0f5faf02912451d12dba07e

libxrdp/xrdp_rdp.c

@@ -705,8 +705,8 @@ xrdp_rdp_send_data_update_sync(struct xrdp_rdp *self)
         LLOGLN(10, ("xrdp_rdp_send_data_update_sync: fastpath"));
         if (xrdp_rdp_init_fastpath(self, s) != 0)
         {
-			free_stream(s);
+            free_stream(s);
-			return 1;
+            return 1;
         }
     }
     else /* slowpath */
@@ -728,7 +728,7 @@ xrdp_rdp_send_data_update_sync(struct xrdp_rdp *self)
         if (xrdp_rdp_send_fastpath(self, s,
                                    FASTPATH_UPDATETYPE_SYNCHRONIZE) != 0)
         {
-			free_stream(s);
+            free_stream(s);
             return 1;
         }
     }

xorg/server/module/rdpCapture.c

@@ -39,9 +39,216 @@
 #define LLOGLN(_level, _args) \
     do { if (_level < LOG_LEVEL) { ErrorF _args ; ErrorF("\n"); } } while (0)
 
+#define RDP_MAX_TILES 1024
+
+/******************************************************************************/
+static int
+rdpLimitRects(RegionPtr reg, int max_rects, BoxPtr *rects)
+{
+    int nrects;
+
+    nrects = REGION_NUM_RECTS(reg);
+    if (nrects > max_rects)
+    {
+        nrects = 1;
+        *rects = rdpRegionExtents(reg);
+    }
+    else
+    {
+        *rects = REGION_RECTS(reg);
+    }
+    return nrects;
+}
+
+/******************************************************************************/
+/* copy rects with no error checking */
+static int
+rdpCopyBox_a8r8g8b8_to_a8r8g8b8(void *src, int src_stride, int srcx, int srcy,
+                                void *dst, int dst_stride, int dstx, int dsty,
+                                BoxPtr rects, int num_rects)
+{
+    char *s8;
+    char *d8;
+    int index;
+    int jndex;
+    int bytes;
+    int height;
+    BoxPtr box;
+
+    for (index = 0; index < num_rects; index++)
+    {
+        box = rects + index;
+        s8 = ((char *) src) + (box->y1 - srcy) * src_stride;
+        s8 += (box->x1 - srcx) * 4;
+        d8 = ((char *) dst) + (box->y1 - dsty) * dst_stride;
+        d8 += (box->x1 - dstx) * 4;
+        bytes = box->x2 - box->x1;
+        bytes *= 4;
+        height = box->y2 - box->y1;
+        for (jndex = 0; jndex < height; jndex++)
+        {
+            memcpy(d8, s8, bytes);
+            d8 += dst_stride;
+            s8 += src_stride;
+        }
+    }
+    return 0;
+}
+
+/******************************************************************************/
+static int
+rdpFillBox_yuvalp(int ax, int ay,
+                 void *dst, int dst_stride)
+{
+    dst = ((char *) dst) + (ay << 8) * (dst_stride >> 8) + (ax << 8);
+    memset(dst, 0, 64 * 64 * 4);
+    return 0;
+}
+
+/******************************************************************************/
+/* copy rects with no error checking
+ * convert ARGB32 to 64x64 linear planar YUVA */
+/* http://msdn.microsoft.com/en-us/library/ff635643.aspx
+ * 0.299   -0.168935    0.499813
+ * 0.587   -0.331665   -0.418531
+ * 0.114    0.50059    -0.081282
+   y = r *  0.299000 + g *  0.587000 + b *  0.114000;
+   u = r * -0.168935 + g * -0.331665 + b *  0.500590;
+   v = r *  0.499813 + g * -0.418531 + b * -0.081282; */
+/* 19595  38470   7471
+  -11071 -21736  32807
+   32756 -27429  -5327 */
+static int
+rdpCopyBox_a8r8g8b8_to_yuvalp(int ax, int ay,
+                              void *src, int src_stride,
+                              void *dst, int dst_stride,
+                              BoxPtr rects, int num_rects)
+{
+    char *s8;
+    char *d8;
+    char *yptr;
+    char *uptr;
+    char *vptr;
+    char *aptr;
+    int *s32;
+    int index;
+    int jndex;
+    int kndex;
+    int width;
+    int height;
+    int pixel;
+    int a;
+    int r;
+    int g;
+    int b;
+    int y;
+    int u;
+    int v;
+    BoxPtr box;
+
+    dst = ((char *) dst) + (ay << 8) * (dst_stride >> 8) + (ax << 8);
+    for (index = 0; index < num_rects; index++)
+    {
+        box = rects + index;
+        s8 = ((char *) src) + box->y1 * src_stride;
+        s8 += box->x1 * 4;
+        d8 = ((char *) dst) + (box->y1 - ay) * 64;
+        d8 += box->x1 - ax;
+        width = box->x2 - box->x1;
+        height = box->y2 - box->y1;
+        for (jndex = 0; jndex < height; jndex++)
+        {
+            s32 = (int *) s8;
+            yptr = d8;
+            uptr = yptr + 64 * 64;
+            vptr = uptr + 64 * 64;
+            aptr = vptr + 64 * 64;
+            kndex = 0;
+            while (kndex < width)
+            {
+                pixel = *(s32++);
+                a = (pixel >> 24) & 0xff;
+                r = (pixel >> 16) & 0xff;
+                g = (pixel >>  8) & 0xff;
+                b = (pixel >>  0) & 0xff;
+                y = (r *  19595 + g *  38470 + b *   7471) >> 16;
+                u = (r * -11071 + g * -21736 + b *  32807) >> 16;
+                v = (r *  32756 + g * -27429 + b *  -5327) >> 16;
+                y = y - 128;
+                y = max(y, -128);
+                u = max(u, -128);
+                v = max(v, -128);
+                y = min(y, 127);
+                u = min(u, 127);
+                v = min(v, 127);
+                *(yptr++) = y;
+                *(uptr++) = u;
+                *(vptr++) = v;
+                *(aptr++) = a;
+                kndex++;
+            }
+            d8 += 64;
+            s8 += src_stride;
+        }
+    }
+    return 0;
+}
+
+/******************************************************************************/
+/* copy rects with no error checking */
+static int
+rdpCopyBox_a8r8g8b8_to_a8b8g8r8(void *src, int src_stride,
+                                void *dst, int dst_stride,
+                                BoxPtr rects, int num_rects)
+{
+    char *s8;
+    char *d8;
+    int index;
+    int jndex;
+    int kndex;
+    int bytes;
+    int width;
+    int height;
+    int red;
+    int green;
+    int blue;
+    BoxPtr box;
+    unsigned int *s32;
+    unsigned int *d32;
+
+    for (index = 0; index < num_rects; index++)
+    {
+        box = rects + index;
+        s8 = ((char *) src) + box->y1 * src_stride;
+        s8 += box->x1 * 4;
+        d8 = ((char *) dst) + box->y1 * dst_stride;
+        d8 += box->x1 * 4;
+        bytes = box->x2 - box->x1;
+        bytes *= 4;
+        width = box->x2 - box->x1;
+        height = box->y2 - box->y1;
+        for (jndex = 0; jndex < height; jndex++)
+        {
+            s32 = (unsigned int *) s8;
+            d32 = (unsigned int *) d8;
+            for (kndex = 0; kndex < width; kndex++)
+            {
+                SPLITCOLOR32(red, green, blue, *s32);
+                *d32 = COLOR24(red, green, blue);
+                s32++;
+                d32++;
+            }
+            d8 += dst_stride;
+            s8 += src_stride;
+        }
+    }
+    return 0;
+}
+
 /******************************************************************************/
 static Bool
-rdpCapture0(RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
+rdpCapture0(rdpClientCon *clientCon,
+            RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
             void *src, int src_width, int src_height,
             int src_stride, int src_format,
             void *dst, int dst_width, int dst_height,
@@ -52,15 +259,13 @@ rdpCapture0(RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
     RegionRec reg;
     char *src_rect;
     char *dst_rect;
-    int num_regions;
+    int num_rects;
-    int bytespp;
     int src_bytespp;
     int dst_bytespp;
     int width;
     int height;
     int src_offset;
     int dst_offset;
-    int bytes;
     int i;
     int j;
     int k;
@@ -69,7 +274,6 @@ rdpCapture0(RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
     int blue;
     Bool rv;
     unsigned int *s32;
-    unsigned int *d32;
     unsigned short *d16;
     unsigned char *d8;
 
@@ -84,27 +288,18 @@ rdpCapture0(RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
     rdpRegionInit(&reg, &rect, 0);
     rdpRegionIntersect(&reg, in_reg, &reg);
 
-    num_regions = REGION_NUM_RECTS(&reg);
+    psrc_rects = 0;
-
+    num_rects = rdpLimitRects(&reg, max_rects, &psrc_rects);
-    if (num_regions > max_rects)
+    if (num_rects < 1)
-    {
-        num_regions = 1;
-        psrc_rects = rdpRegionExtents(&reg);
-    }
-    else
-    {
-        psrc_rects = REGION_RECTS(&reg);
-    }
-
-    if (num_regions < 1)
     {
+        rdpRegionUninit(&reg);
         return FALSE;
     }
 
-    *num_out_rects = num_regions;
+    *num_out_rects = num_rects;
 
-    *out_rects = (BoxPtr) g_malloc(sizeof(BoxRec) * num_regions, 0);
+    *out_rects = (BoxPtr) g_malloc(sizeof(BoxRec) * num_rects, 0);
-    for (i = 0; i < num_regions; i++)
+    for (i = 0; i < num_rects; i++)
     {
         rect = psrc_rects[i];
         (*out_rects)[i] = rect;
@@ -112,78 +307,22 @@ rdpCapture0(RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
 
     if ((src_format == XRDP_a8r8g8b8) && (dst_format == XRDP_a8r8g8b8))
     {
-        bytespp = 4;
+        rdpCopyBox_a8r8g8b8_to_a8r8g8b8(src, src_stride, 0, 0,
-
+                                        dst, dst_stride, 0, 0,
-        for (i = 0; i < num_regions; i++)
+                                        psrc_rects, num_rects);
-        {
-            /* get rect to copy */
-            rect = (*out_rects)[i];
-
-            /* get rect dimensions */
-            width = rect.x2 - rect.x1;
-            height = rect.y2 - rect.y1;
-
-            /* point to start of each rect in respective memory */
-            src_offset = rect.y1 * src_stride + rect.x1 * bytespp;
-            dst_offset = rect.y1 * dst_stride + rect.x1 * bytespp;
-            src_rect = src + src_offset;
-            dst_rect = dst + dst_offset;
-
-            /* bytes per line */
-            bytes = width * bytespp;
-
-            /* copy one line at a time */
-            for (j = 0; j < height; j++)
-            {
-                memcpy(dst_rect, src_rect, bytes);
-                src_rect += src_stride;
-                dst_rect += dst_stride;
-            }
-        }
     }
     else if ((src_format == XRDP_a8r8g8b8) && (dst_format == XRDP_a8b8g8r8))
     {
-        src_bytespp = 4;
+        rdpCopyBox_a8r8g8b8_to_a8b8g8r8(src, src_stride,
-        dst_bytespp = 4;
+                                        dst, dst_stride,
-
+                                        psrc_rects, num_rects);
-        for (i = 0; i < num_regions; i++)
-        {
-            /* get rect to copy */
-            rect = (*out_rects)[i];
-
-            /* get rect dimensions */
-            width = rect.x2 - rect.x1;
-            height = rect.y2 - rect.y1;
-
-            /* point to start of each rect in respective memory */
-            src_offset = rect.y1 * src_stride + rect.x1 * src_bytespp;
-            dst_offset = rect.y1 * dst_stride + rect.x1 * dst_bytespp;
-            src_rect = src + src_offset;
-            dst_rect = dst + dst_offset;
-
-            /* copy one line at a time */
-            for (j = 0; j < height; j++)
-            {
-                s32 = (unsigned int *) src_rect;
-                d32 = (unsigned int *) dst_rect;
-                for (k = 0; k < width; k++)
-                {
-                    SPLITCOLOR32(red, green, blue, *s32);
-                    *d32 = COLOR24(red, green, blue);
-                    s32++;
-                    d32++;
-                }
-                src_rect += src_stride;
-                dst_rect += dst_stride;
-            }
-        }
     }
     else if ((src_format == XRDP_a8r8g8b8) && (dst_format == XRDP_r5g6b5))
     {
         src_bytespp = 4;
         dst_bytespp = 2;
 
-        for (i = 0; i < num_regions; i++)
+        for (i = 0; i < num_rects; i++)
         {
             /* get rect to copy */
             rect = (*out_rects)[i];
@@ -220,7 +359,7 @@ rdpCapture0(RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
         src_bytespp = 4;
         dst_bytespp = 2;
 
-        for (i = 0; i < num_regions; i++)
+        for (i = 0; i < num_rects; i++)
         {
             /* get rect to copy */
             rect = (*out_rects)[i];
@@ -257,7 +396,7 @@ rdpCapture0(RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
         src_bytespp = 4;
         dst_bytespp = 1;
 
-        for (i = 0; i < num_regions; i++)
+        for (i = 0; i < num_rects; i++)
         {
             /* get rect to copy */
             rect = (*out_rects)[i];
@@ -300,7 +439,8 @@ rdpCapture0(RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
 /******************************************************************************/
 /* make out_rects always multiple of 16 width and height */
 static Bool
-rdpCapture1(RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
+rdpCapture1(rdpClientCon *clientCon,
+            RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
             void *src, int src_width, int src_height,
             int src_stride, int src_format,
             void *dst, int dst_width, int dst_height,
@@ -475,11 +615,125 @@ rdpCapture1(RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
     return rv;
 }
 
+/******************************************************************************/
+static Bool
+rdpCapture2(rdpClientCon *clientCon,
+            RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
+            void *src, int src_width, int src_height,
+            int src_stride, int src_format,
+            void *dst, int dst_width, int dst_height,
+            int dst_stride, int dst_format, int max_rects)
+{
+    int x;
+    int y;
+    int out_rect_index;
+    int num_rects;
+    int rcode;
+    BoxRec rect;
+    BoxRec extents_rect;
+    BoxPtr rects;
+    RegionRec tile_reg;
+    RegionRec lin_reg;
+    RegionRec temp_reg;
+    RegionPtr pin_reg;
+
+    LLOGLN(10, ("rdpCapture2:"));
+
+    *out_rects = (BoxPtr) g_malloc(sizeof(BoxRec) * RDP_MAX_TILES, 0);
+    if (*out_rects == NULL)
+    {
+        return FALSE;
+    }
+    out_rect_index = 0;
+
+    /* clip for smaller of 2 */
+    rect.x1 = 0;
+    rect.y1 = 0;
+    rect.x2 = min(dst_width, src_width);
+    rect.y2 = min(dst_height, src_height);
+    rdpRegionInit(&temp_reg, &rect, 0);
+    rdpRegionIntersect(&temp_reg, in_reg, &temp_reg);
+
+    /* limit the numer of rects */
+    num_rects = REGION_NUM_RECTS(&temp_reg);
+    if (num_rects > max_rects)
+    {
+        LLOGLN(10, ("rdpCapture2: too many rects"));
+        rdpRegionInit(&lin_reg, rdpRegionExtents(&temp_reg), 0);
+        pin_reg = &lin_reg;
+    }
+    else
+    {
+        LLOGLN(10, ("rdpCapture2: not too many rects"));
+        rdpRegionInit(&lin_reg, NullBox, 0);
+        pin_reg = &temp_reg;
+    }
+
+    extents_rect = *rdpRegionExtents(pin_reg);
+    y = extents_rect.y1 & ~63;
+    while (y < extents_rect.y2)
+    {
+        x = extents_rect.x1 & ~63;
+        while (x < extents_rect.x2)
+        {
+            rect.x1 = x;
+            rect.y1 = y;
+            rect.x2 = rect.x1 + 64;
+            rect.y2 = rect.y1 + 64;
+            rcode = rdpRegionContainsRect(pin_reg, &rect);
+            LLOGLN(10, ("rdpCapture2: rcode %d", rcode));
+
+            if (rcode != rgnOUT)
+            {
+                if (rcode == rgnPART)
+                {
+                    LLOGLN(10, ("rdpCapture2: rgnPART"));
+                    rdpFillBox_yuvalp(x, y, dst, dst_stride);
+                    rdpRegionInit(&tile_reg, &rect, 0);
+                    rdpRegionIntersect(&tile_reg, pin_reg, &tile_reg);
+                    rects = REGION_RECTS(&tile_reg);
+                    num_rects = REGION_NUM_RECTS(&tile_reg);
+                    rdpCopyBox_a8r8g8b8_to_yuvalp(x, y,
+                                                  src, src_stride,
+                                                  dst, dst_stride,
+                                                  rects, num_rects);
+                    rdpRegionUninit(&tile_reg);
+                }
+                else /* rgnIN */
+                {
+                    LLOGLN(10, ("rdpCapture2: rgnIN"));
+                    rdpCopyBox_a8r8g8b8_to_yuvalp(x, y,
+                                                  src, src_stride,
+                                                  dst, dst_stride,
+                                                  &rect, 1);
+                }
+                (*out_rects)[out_rect_index] = rect;
+                out_rect_index++;
+                if (out_rect_index >= RDP_MAX_TILES)
+                {
+                    g_free(*out_rects);
+                    *out_rects = NULL;
+                    rdpRegionUninit(&temp_reg);
+                    rdpRegionUninit(&lin_reg);
+                    return FALSE;
+                }
+            }
+            x += 64;
+        }
+        y += 64;
+    }
+    *num_out_rects = out_rect_index;
+    rdpRegionUninit(&temp_reg);
+    rdpRegionUninit(&lin_reg);
+    return TRUE;
+}
+
 /**
  * Copy an array of rectangles from one memory area to another
  *****************************************************************************/
 Bool
-rdpCapture(RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
+rdpCapture(rdpClientCon *clientCon,
+           RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
            void *src, int src_width, int src_height,
            int src_stride, int src_format,
            void *dst, int dst_width, int dst_height,
@@ -489,13 +743,19 @@ rdpCapture(RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
     switch (mode)
     {
         case 0:
-            return rdpCapture0(in_reg, out_rects, num_out_rects,
+            return rdpCapture0(clientCon, in_reg, out_rects, num_out_rects,
                                src, src_width, src_height,
                                src_stride, src_format,
                                dst, dst_width, dst_height,
                                dst_stride, dst_format, 15);
         case 1:
-            return rdpCapture1(in_reg, out_rects, num_out_rects,
+            return rdpCapture1(clientCon, in_reg, out_rects, num_out_rects,
+                               src, src_width, src_height,
+                               src_stride, src_format,
+                               dst, dst_width, dst_height,
+                               dst_stride, dst_format, 15);
+        case 2:
+            return rdpCapture2(clientCon, in_reg, out_rects, num_out_rects,
                                src, src_width, src_height,
                                src_stride, src_format,
                                dst, dst_width, dst_height,

xorg/server/module/rdpCapture.h

@@ -19,7 +19,8 @@
  */
 
 Bool
-rdpCapture(RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
+rdpCapture(rdpClientCon *clientCon,
+           RegionPtr in_reg, BoxPtr *out_rects, int *num_out_rects,
            void *src, int src_width, int src_height,
            int src_stride, int src_format,
            void *dst, int dst_width, int dst_height,

xorg/server/module/rdpClientCon.c

@@ -691,11 +691,9 @@ rdpClientConProcessMsgClientInfo(rdpPtr dev, rdpClientCon *clientCon)
     i1 = clientCon->client_info.offscreen_cache_entries;
     LLOGLN(0, ("  offscreen entries %d", i1));
 
-    if ((clientCon->client_info.mcs_connection_type == 6) && /* LAN */
+    if (clientCon->client_info.capture_format != 0)
-        (clientCon->client_info.jpeg_codec_id == 2))
     {
-        /* jpeg capture needs swap */
+        clientCon->rdp_format = clientCon->client_info.capture_format;
-        clientCon->rdp_format = XRDP_a8b8g8r8;
     }
 
     if (clientCon->client_info.offscreen_support_level > 0)
@@ -1964,7 +1962,9 @@ rdpDeferredUpdateCallback(OsTimerPtr timer, CARD32 now, pointer arg)
     LLOGLN(10, ("rdpDeferredUpdateCallback:"));
     clientCon = (rdpClientCon *) arg;
 
-    if (clientCon->rect_id > clientCon->rect_id_ack)
+    if ((clientCon->rect_id > clientCon->rect_id_ack) ||
+        /* do not allow captures until we have the client_info */
+        clientCon->client_info.size == 0)
     {
         LLOGLN(0, ("rdpDeferredUpdateCallback: reschedual rect_id %d "
                "rect_id_ack %d",
@@ -1986,13 +1986,16 @@ rdpDeferredUpdateCallback(OsTimerPtr timer, CARD32 now, pointer arg)
     clientCon->updateSchedualed = FALSE;
     rects = 0;
     num_rects = 0;
-    if (rdpCapture(clientCon->dirtyRegion, &rects, &num_rects,
+    LLOGLN(10, ("rdpDeferredUpdateCallback: capture_code %d",
+           clientCon->client_info.capture_code));
+    if (rdpCapture(clientCon, clientCon->dirtyRegion, &rects, &num_rects,
                    id.pixels, id.width, id.height,
                    id.lineBytes, XRDP_a8r8g8b8, id.shmem_pixels,
                    clientCon->rdp_width, clientCon->rdp_height,
                    clientCon->rdp_width * clientCon->rdp_Bpp,
-                   clientCon->rdp_format, 0))
+                   clientCon->rdp_format, clientCon->client_info.capture_code))
     {
+        LLOGLN(10, ("rdpDeferredUpdateCallback: num_rects %d", num_rects));
         rdpClientConSendPaintRectShmEx(clientCon->dev, clientCon, &id,
                                        clientCon->dirtyRegion,
                                        rects, num_rects);

xrdp/xrdp_encoder.c

@@ -39,19 +39,28 @@
   } \
   while (0)
 
+#define JPG_CODEC 0
+#define RFX_CODEC 1
+
+/*****************************************************************************/
+static int
+process_enc_jpg(struct xrdp_mm *self, XRDP_ENC_DATA *enc);
+static int
+process_enc_rfx(struct xrdp_mm *self, XRDP_ENC_DATA *enc);
+
 /**
  * Init encoder
  *
  * @return 0 on success, -1 on failure
  *****************************************************************************/
-
+/* called from main thread */
 int APP_CC
 init_xrdp_encoder(struct xrdp_mm *self)
 {
     char buf[1024];
     int pid;
 
-    LLOGLN(0, ("init_xrdp_encoder: initing encoder"));
+    LLOGLN(0, ("init_xrdp_encoder: initing encoder codec_id %d", self->codec_id));
 
     if (self == 0)
     {
@@ -72,6 +81,28 @@ init_xrdp_encoder(struct xrdp_mm *self)
     g_snprintf(buf, 1024, "xrdp_%8.8x_encoder_term", pid);
     self->xrdp_encoder_term = g_create_wait_obj(buf);
 
+    switch (self->codec_id)
+    {
+        case 2:
+            self->process_enc = process_enc_jpg;
+            break;
+        case 3:
+            self->process_enc = process_enc_rfx;
+#ifdef XRDP_RFXCODEC
+            self->codec_handle =
+                    rfxcodec_encode_create(self->wm->screen->width,
+                                           self->wm->screen->height,
+                                           RFX_FORMAT_YUV, 0);
+                                           //RFX_FORMAT_BGRA, 0);
+#endif
+            break;
+        default:
+            LLOGLN(0, ("init_xrdp_encoder: unknown codec_id %d",
+                   self->codec_id));
+            break;
+            
+    }
+
     /* create thread to process messages */
     tc_thread_create(proc_enc_msg, self);
 
@@ -104,6 +135,13 @@ deinit_xrdp_encoder(struct xrdp_mm *self)
     g_set_wait_obj(self->xrdp_encoder_term);
     g_sleep(1000);
 
+    if (self->codec_id == 3)
+    {
+#ifdef XRDP_RFXCODEC
+        rfxcodec_encode_destroy(self->codec_handle);
+#endif
+    }
+
     /* destroy wait objects used for signalling */
     g_delete_wait_obj(self->xrdp_encoder_event_to_proc);
     g_delete_wait_obj(self->xrdp_encoder_event_processed);
@@ -147,8 +185,9 @@ deinit_xrdp_encoder(struct xrdp_mm *self)
 }
 
 /*****************************************************************************/
+/* called from encoder thread */
 static int
-process_enc(struct xrdp_mm *self, XRDP_ENC_DATA *enc)
+process_enc_jpg(struct xrdp_mm *self, XRDP_ENC_DATA *enc)
 {
     int                  index;
     int                  x;
@@ -165,7 +204,7 @@ process_enc(struct xrdp_mm *self, XRDP_ENC_DATA *enc)
     tbus                 mutex;
     tbus                 event_processed;
 
-    LLOGLN(10, ("process_enc:"));
+    LLOGLN(10, ("process_enc_jpg:"));
     quality = self->codec_quality;
     fifo_processed = self->fifo_processed;
     mutex = self->mutex;
@@ -179,19 +218,22 @@ process_enc(struct xrdp_mm *self, XRDP_ENC_DATA *enc)
         cy = enc->crects[index * 4 + 3];
         if (cx < 1 || cy < 1)
         {
-            LLOGLN(0, ("process_enc: error 1"));
+            LLOGLN(0, ("process_enc_jpg: error 1"));
             continue;
         }
+
+        LLOGLN(10, ("process_enc_jpg: x %d y %d cx %d cy %d", x, y, cx, cy));
+
         out_data_bytes = MAX((cx + 4) * cy * 4, 8192);
         if ((out_data_bytes < 1) || (out_data_bytes > 16 * 1024 * 1024))
         {
-            LLOGLN(0, ("process_enc: error 2"));
+            LLOGLN(0, ("process_enc_jpg: error 2"));
             return 1;
         }
         out_data = (char *) g_malloc(out_data_bytes + 256 + 2, 0);
         if (out_data == 0)
         {
-            LLOGLN(0, ("process_enc: error 3"));
+            LLOGLN(0, ("process_enc_jpg: error 3"));
             return 1;
         }
         out_data[256] = 0; /* header bytes */
@@ -203,7 +245,7 @@ process_enc(struct xrdp_mm *self, XRDP_ENC_DATA *enc)
                                             out_data + 256 + 2, &out_data_bytes);
         if (error < 0)
         {
-            LLOGLN(0, ("process_enc: jpeg error %d bytes %d",
+            LLOGLN(0, ("process_enc_jpg: jpeg error %d bytes %d",
                    error, out_data_bytes));
             g_free(out_data);
             return 1;
@@ -216,7 +258,10 @@ process_enc(struct xrdp_mm *self, XRDP_ENC_DATA *enc)
         enc_done->comp_pad_data = out_data;
         enc_done->enc = enc;
         enc_done->last = index == (enc->num_crects - 1);
-        enc_done->index = index;
+        enc_done->x = x;
+        enc_done->y = y;
+        enc_done->cx = cx;
+        enc_done->cy = cy;
         /* done with msg */
         /* inform main thread done */
         tc_mutex_lock(mutex);
@@ -228,10 +273,125 @@ process_enc(struct xrdp_mm *self, XRDP_ENC_DATA *enc)
     return 0;
 }
 
+#ifdef XRDP_RFXCODEC
+
+/*****************************************************************************/
+/* called from encoder thread */
+static int
+process_enc_rfx(struct xrdp_mm *self, XRDP_ENC_DATA *enc)
+{
+    int index;
+    int x;
+    int y;
+    int cx;
+    int cy;
+    int out_data_bytes;
+    int count;
+    int error;
+    char *out_data;
+    XRDP_ENC_DATA_DONE *enc_done;
+    FIFO *fifo_processed;
+    tbus mutex;
+    tbus event_processed;
+    struct rfx_tile *tiles;
+    struct rfx_rect *rfxrects;
+
+    LLOGLN(10, ("process_enc_rfx:"));
+    LLOGLN(10, ("process_enc_rfx: num_crects %d num_drects %d",
+           enc->num_crects, enc->num_drects));
+    fifo_processed = self->fifo_processed;
+    mutex = self->mutex;
+    event_processed = self->xrdp_encoder_event_processed;
+
+    if ((enc->num_crects > 512) || (enc->num_drects > 512))
+    {
+        return 0;
+    }
+
+    out_data_bytes = 16 * 1024 * 1024;
+    index = 256 + sizeof(struct rfx_tile) * 512 +
+                  sizeof(struct rfx_rect) * 512;
+    out_data = (char *) g_malloc(out_data_bytes + index, 0);
+    if (out_data == 0)
+    {
+        return 0;
+    }
+    tiles = (struct rfx_tile *) (out_data + out_data_bytes + 256);
+    rfxrects = (struct rfx_rect *) (tiles + 512);
+
+    count = enc->num_crects;
+    for (index = 0; index < count; index++)
+    {
+        x = enc->crects[index * 4 + 0];
+        y = enc->crects[index * 4 + 1];
+        cx = enc->crects[index * 4 + 2];
+        cy = enc->crects[index * 4 + 3];
+        LLOGLN(10, ("process_enc_rfx:"));
+        tiles[index].x = x;
+        tiles[index].y = y;
+        tiles[index].cx = cx;
+        tiles[index].cy = cy;
+        LLOGLN(10, ("x %d y %d cx %d cy %d", x, y, cx, cy));
+        tiles[index].quant_y = 0;
+        tiles[index].quant_cb = 0;
+        tiles[index].quant_cr = 0;
+    }
+
+    count = enc->num_drects;
+    for (index = 0; index < count; index++)
+    {
+        x = enc->drects[index * 4 + 0];
+        y = enc->drects[index * 4 + 1];
+        cx = enc->drects[index * 4 + 2];
+        cy = enc->drects[index * 4 + 3];
+        LLOGLN(10, ("process_enc_rfx:"));
+        rfxrects[index].x = x;
+        rfxrects[index].y = y;
+        rfxrects[index].cx = cx;
+        rfxrects[index].cy = cy;
+    }
+
+    error = rfxcodec_encode(self->codec_handle, out_data + 256, &out_data_bytes,
+                            enc->data, enc->width, enc->height, enc->width * 4,
+                            rfxrects, enc->num_drects,
+                            tiles, enc->num_crects, 0, 0);
+    LLOGLN(10, ("process_enc_rfx: rfxcodec_encode rv %d", error));
+
+    enc_done = (XRDP_ENC_DATA_DONE *)
+               g_malloc(sizeof(XRDP_ENC_DATA_DONE), 1);
+    enc_done->comp_bytes = out_data_bytes;
+    enc_done->pad_bytes = 256;
+    enc_done->comp_pad_data = out_data;
+    enc_done->enc = enc;
+    enc_done->last = 1;
+    enc_done->cx = self->wm->screen->width;
+    enc_done->cy = self->wm->screen->height;
+
+    /* done with msg */
+    /* inform main thread done */
+    tc_mutex_lock(mutex);
+    fifo_add_item(fifo_processed, enc_done);
+    tc_mutex_unlock(mutex);
+    /* signal completion for main thread */
+    g_set_wait_obj(event_processed);
+
+    return 0;
+}
+
+#else
+
+/*****************************************************************************/
+/* called from encoder thread */
+static int
+process_enc_rfx(struct xrdp_mm *self, XRDP_ENC_DATA *enc)
+{
+    return 0;
+}
+
+#endif
+
 /**
- * Init encoder
+ * Encoder thread main loop
- *
- * @return 0 on success, -1 on failure
  *****************************************************************************/
 THREAD_RV THREAD_CC
 proc_enc_msg(void *arg)
@@ -305,7 +465,7 @@ proc_enc_msg(void *arg)
             while (enc != 0)
             {
                 /* do work */
-                process_enc(self, enc);
+                self->process_enc(self, enc);
                 /* get next msg */
                 tc_mutex_lock(mutex);
                 enc = (XRDP_ENC_DATA *) fifo_remove_item(fifo_to_proc);

xrdp/xrdp_mm.c

@@ -55,7 +55,13 @@ xrdp_mm_create(struct xrdp_wm *owner)
     self->login_values = list_create();
     self->login_values->auto_free = 1;
 
-    LLOGLN(10, ("xrdp_mm_create: bpp %d", self->wm->client_info->bpp));
+    LLOGLN(0, ("xrdp_mm_create: bpp %d mcs_connection_type %d "
+           "jpeg_codec_id %d v3_codec_id %d rfx_codec_id %d",
+           self->wm->client_info->bpp,
+           self->wm->client_info->mcs_connection_type,
+           self->wm->client_info->jpeg_codec_id,
+           self->wm->client_info->v3_codec_id,
+           self->wm->client_info->rfx_codec_id));
     /* go into jpeg codec mode if jpeg set, lan set */
     if (self->wm->client_info->mcs_connection_type == 6) /* LAN */
     {
@@ -67,6 +73,20 @@ xrdp_mm_create(struct xrdp_wm *owner)
                 self->codec_id = 2;
                 self->in_codec_mode = 1;
                 self->codec_quality = self->wm->client_info->jpeg_prop[0];
+                self->wm->client_info->capture_code = 0;
+                self->wm->client_info->capture_format =
+                /* PIXMAN_a8b8g8r8 */
+                (32 << 24) | (3 << 16) | (8 << 12) | (8 << 8) | (8 << 4) | 8;
+            }
+        }
+        else if (self->wm->client_info->rfx_codec_id == 3) /* RFX */
+        {
+            if (self->wm->client_info->bpp > 16)
+            {
+                LLOGLN(0, ("xrdp_mm_create: starting rfx codec session"));
+                self->codec_id = 3;
+                self->in_codec_mode = 1;
+                self->wm->client_info->capture_code = 2;
             }
         }
     }
@@ -2067,27 +2087,31 @@ xrdp_mm_check_wait_objs(struct xrdp_mm *self)
                 LLOGLN(10, ("xrdp_mm_check_wait_objs: message back bytes %d",
                        enc_done->comp_bytes));
 
-                x = enc_done->enc->crects[enc_done->index * 4 + 0];
+                x = enc_done->x;
-                y = enc_done->enc->crects[enc_done->index * 4 + 1];
+                y = enc_done->y;
-                cx = enc_done->enc->crects[enc_done->index * 4 + 2];
+                cx = enc_done->cx;
-                cy = enc_done->enc->crects[enc_done->index * 4 + 3];
+                cy = enc_done->cy;
 
 #if DUMP_JPEG
                 xrdp_mm_dump_jpeg(self, enc_done);
 #endif
 
-                libxrdp_fastpath_send_surface(self->wm->session,
+                if (enc_done->comp_bytes > 0)
-                                              enc_done->comp_pad_data,
+                {
-                                              enc_done->pad_bytes,
+                    libxrdp_fastpath_send_surface(self->wm->session,
-                                              enc_done->comp_bytes,
+                                                  enc_done->comp_pad_data,
-                                              x, y, x + cx, y + cy,
+                                                  enc_done->pad_bytes,
-                                              32, 2, cx, cy);
+                                                  enc_done->comp_bytes,
+                                                  x, y, x + cx, y + cy,
+                                                  32, self->codec_id, cx, cy);
+                }
 
                 /* free enc_done */
                 if (enc_done->last)
                 {
                     LLOGLN(10, ("xrdp_mm_check_wait_objs: last set"));
-                    self->mod->mod_frame_ack(self->mod, enc_done->enc->flags, enc_done->enc->frame_id);
+                           self->mod->mod_frame_ack(self->mod,
+                           enc_done->enc->flags, enc_done->enc->frame_id);
                     g_free(enc_done->enc->drects);
                     g_free(enc_done->enc->crects);
                     g_free(enc_done->enc);

xrdp/xrdp_types.h

@@ -268,6 +268,9 @@ struct xrdp_cache
   struct list* xrdp_os_del_list;
 };
 
+/* defined later */
+struct xrdp_enc_data;
+
 struct xrdp_mm
 {
   struct xrdp_wm* wm; /* owner */
@@ -300,6 +303,8 @@ struct xrdp_mm
   FIFO *fifo_to_proc;
   FIFO *fifo_processed;
   tbus  mutex;
+  int (*process_enc)(struct xrdp_mm *self, struct xrdp_enc_data *enc);
+  void *codec_handle;
 };
 
 struct xrdp_key_info
@@ -641,7 +646,10 @@ struct xrdp_enc_data_done
     char                 *comp_pad_data;
     struct xrdp_enc_data *enc;
     int                   last; /* true is this is last message for enc */
-    int                   index; /* depends on codec */
+    int                   x;
+    int                   y;
+    int                   cx;
+    int                   cy;
 };
 
 typedef struct xrdp_enc_data_done XRDP_ENC_DATA_DONE;

xup/xup.c

@@ -1380,6 +1380,7 @@ lib_send_client_info(struct mod *mod)
     struct stream *s;
     int len;
 
+    g_writeln("lib_send_client_info:");
     make_stream(s);
     init_stream(s, 8192);
     s_push_layer(s, iso_hdr, 4);