infosecn1nja
/
metasploit-framework
mirror of https://github.com/infosecn1nja/metasploit-framework.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity
metasploit-framework/external/source/vncdll/winvnc/vncEncodeTight.cpp

1619 lines
45 KiB
C++
Raw Normal View History

Commit the new VNC x86/x64 DLL source code... git-svn-id: file:///home/svn/framework3/trunk@8745 4d416f70-5f16-0410-b530-b9f4589650da
2010-03-08 14:49:45 +00:00
// Copyright (C) 2000 Constantin Kaplinsky. All Rights Reserved.
// Copyright (C) 2000 Tridia Corporation. All Rights Reserved.
// Copyright (C) 1999 AT&T Laboratories Cambridge. All Rights Reserved.
//
// This file is part of the VNC system.
//
// The VNC system is free software; you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation; either version 2 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program; if not, write to the Free Software
// Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307,
// USA.
//
// TightVNC distribution homepage on the Web: http://www.tightvnc.com/
//
// If the source code for the VNC system is not available from the place
// whence you received this file, check http://www.uk.research.att.com/vnc or contact
// the authors on vnc@uk.research.att.com for information on obtaining it.
// vncEncodeTight
// This file implements the vncEncoder-derived vncEncodeTight class.
// This class overrides some vncEncoder functions to produce a bitmap
// to Tight encoder. Tight is much more efficient than RAW format on
// most screen data and usually 2..10 times as efficient as hextile.
// It's also more efficient than Zlib encoding in most cases.
// But note that tight compression may use more CPU time on the server.
// However, over slower (128kbps or less) connections, the reduction
// in data transmitted usually outweighs the extra latency added
// while the server CPU performs the compression algorithms.
#include "vncEncodeTight.h"
// Compression level stuff. The following array contains various
// encoder parameters for each of 10 compression levels (0..9).
// Last three parameters correspond to JPEG quality levels (0..9).
//
// NOTE: m_conf[9].maxRectSize should be >= m_conf[i].maxRectSize,
// where i in [0..8]. RequiredBuffSize() method depends on this.
const TIGHT_CONF vncEncodeTight::m_conf[10] = {
{ 512, 32, 6, 65536, 0, 0, 0, 0, 0, 0, 4, 5, 10000, 23000 },
{ 2048, 128, 6, 65536, 1, 1, 1, 0, 0, 0, 8, 10, 8000, 18000 },
{ 6144, 256, 8, 65536, 3, 3, 2, 0, 0, 0, 24, 15, 6500, 15000 },
{ 10240, 1024, 12, 65536, 5, 5, 3, 0, 0, 0, 32, 25, 5000, 12000 },
{ 16384, 2048, 12, 65536, 6, 6, 4, 0, 0, 0, 32, 37, 4000, 10000 },
{ 32768, 2048, 12, 4096, 7, 7, 5, 4, 150, 380, 32, 50, 3000, 8000 },
{ 65536, 2048, 16, 4096, 7, 7, 6, 4, 170, 420, 48, 60, 2000, 5000 },
{ 65536, 2048, 16, 4096, 8, 8, 7, 5, 180, 450, 64, 70, 1000, 2500 },
{ 65536, 2048, 32, 8192, 9, 9, 8, 6, 190, 475, 64, 75, 500, 1200 },
{ 65536, 2048, 32, 8192, 9, 9, 9, 6, 200, 500, 96, 80, 200, 500 }
};
vncEncodeTight::vncEncodeTight()
{
m_buffer = NULL;
m_bufflen = 0;
m_hdrBuffer = new BYTE [sz_rfbFramebufferUpdateRectHeader + 8 + 256*4];
m_prevRowBuf = NULL;
for (int i = 0; i < 4; i++)
m_zsActive[i] = false;
}
vncEncodeTight::~vncEncodeTight()
{
if (m_buffer != NULL) {
delete[] m_buffer;
m_buffer = NULL;
}
delete[] m_hdrBuffer;
for (int i = 0; i < 4; i++) {
if (m_zsActive[i])
deflateEnd(&m_zsStruct[i]);
m_zsActive[i] = false;
}
}
void
vncEncodeTight::Init()
{
vncEncoder::Init();
}
/*****************************************************************************
*
* Routines to implement Tight Encoding.
*
*/
UINT
vncEncodeTight::RequiredBuffSize(UINT width, UINT height)
{
// FIXME: Use actual compression level instead of 9?
int result = m_conf[9].maxRectSize * (m_remoteformat.bitsPerPixel / 8);
result += result / 100 + 16;
return result;
}
UINT
vncEncodeTight::NumCodedRects(RECT &rect)
{
const int w = rect.right - rect.left;
const int h = rect.bottom - rect.top;
// No matter how many rectangles we will send if LastRect markers
// are used to terminate rectangle stream.
if (m_use_lastrect && w * h >= MIN_SPLIT_RECT_SIZE) {
return 0;
}
const int maxRectSize = m_conf[m_compresslevel].maxRectSize;
const int maxRectWidth = m_conf[m_compresslevel].maxRectWidth;
if (w > maxRectWidth || w * h > maxRectSize) {
const int subrectMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
const int subrectMaxHeight = maxRectSize / subrectMaxWidth;
return (((w - 1) / maxRectWidth + 1) *
((h - 1) / subrectMaxHeight + 1));
} else {
return 1;
}
}
UINT
vncEncodeTight::EncodeRect(BYTE *source, VSocket *outConn, BYTE *dest,
const RECT &rect, int offx, int offy)
{
int x = rect.left, y = rect.top;
int w = rect.right - x, h = rect.bottom - y;
offsetx = offx;
offsety = offy;
const int maxRectSize = m_conf[m_compresslevel].maxRectSize;
const int rawDataSize = maxRectSize * (m_remoteformat.bitsPerPixel / 8);
if (m_bufflen < rawDataSize) {
if (m_buffer != NULL)
delete [] m_buffer;
m_buffer = new BYTE [rawDataSize+1];
if (m_buffer == NULL)
return vncEncoder::EncodeRect(source, dest, rect, offsetx, offsety);
m_bufflen = rawDataSize;
}
if ( m_remoteformat.depth == 24 && m_remoteformat.redMax == 0xFF &&
m_remoteformat.greenMax == 0xFF && m_remoteformat.blueMax == 0xFF ) {
m_usePixelFormat24 = true;
} else {
m_usePixelFormat24 = false;
}
if (!m_use_lastrect || w * h < MIN_SPLIT_RECT_SIZE)
return EncodeRectSimple(source, outConn, dest, rect);
// Calculate maximum number of rows in one non-solid rectangle.
int nMaxRows;
{
int maxRectSize = m_conf[m_compresslevel].maxRectSize;
int maxRectWidth = m_conf[m_compresslevel].maxRectWidth;
int nMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
nMaxRows = maxRectSize / nMaxWidth;
}
// Try to find large solid-color areas and send them separately.
CARD32 colorValue;
int x_best, y_best, w_best, h_best;
int dx, dy, dw, dh;
for (dy = y; dy < y + h; dy += MAX_SPLIT_TILE_SIZE) {
// If a rectangle becomes too large, send its upper part now.
if (dy - y >= nMaxRows) {
RECT upperRect;
SetRect(&upperRect, x, y, x + w, y + nMaxRows);
int size = EncodeRectSimple(source, outConn, dest, upperRect);
outConn->SendQueued((char *)dest, size);
y += nMaxRows;
h -= nMaxRows;
}
dh = (dy + MAX_SPLIT_TILE_SIZE <= y + h) ?
MAX_SPLIT_TILE_SIZE : (y + h - dy);
for (dx = x; dx < x + w; dx += MAX_SPLIT_TILE_SIZE) {
dw = (dx + MAX_SPLIT_TILE_SIZE <= x + w) ?
MAX_SPLIT_TILE_SIZE : (x + w - dx);
if (CheckSolidTile(source, dx, dy, dw, dh, &colorValue, FALSE)) {
// Get dimensions of solid-color area.
FindBestSolidArea(source, dx, dy, w - (dx - x), h - (dy - y),
colorValue, &w_best, &h_best);
// Make sure a solid rectangle is large enough
// (or the whole rectangle is of the same color).
if ( w_best * h_best != w * h &&
w_best * h_best < MIN_SOLID_SUBRECT_SIZE )
continue;
// Try to extend solid rectangle to maximum size.
x_best = dx; y_best = dy;
ExtendSolidArea(source, x, y, w, h, colorValue,
&x_best, &y_best, &w_best, &h_best);
// Compute dimensions of surrounding rectangles.
RECT rects[4];
SetRect(&rects[0],
x, y, x + w, y_best);
SetRect(&rects[1],
x, y_best, x_best, y_best + h_best);
SetRect(&rects[2],
x_best + w_best, y_best, x + w, y_best + h_best);
SetRect(&rects[3],
x, y_best + h_best, x + w, y + h);
// Send solid-color area and surrounding rectangles.
for (int i = 0; i < 4; i++) {
if (i == 2) {
RECT onePixel;
SetRect(&onePixel,
x_best, y_best, x_best + 1, y_best + 1);
Translate(source, m_buffer, onePixel);
SendTightHeader(x_best, y_best, w_best, h_best);
int size = SendSolidRect(dest);
outConn->SendQueued((char *)m_hdrBuffer, m_hdrBufferBytes);
outConn->SendQueued((char *)dest, size);
encodedSize += (m_hdrBufferBytes + size -
sz_rfbFramebufferUpdateRectHeader);
transmittedSize += (m_hdrBufferBytes + size);
}
if ( rects[i].left == rects[i].right ||
rects[i].top == rects[i].bottom ) {
continue;
}
int size = EncodeRect(source, outConn, dest, rects[i], offsetx, offsety);
outConn->SendQueued((char *)dest, size);
}
// Return after all recursive calls done (0 == data sent).
return 0;
}
}
}
// No suitable solid-color rectangles found.
return EncodeRectSimple(source, outConn, dest, rect);
}
void
vncEncodeTight::FindBestSolidArea(BYTE *source, int x, int y, int w, int h,
CARD32 colorValue, int *w_ptr, int *h_ptr)
{
int dx, dy, dw, dh;
int w_prev;
int w_best = 0, h_best = 0;
w_prev = w;
for (dy = y; dy < y + h; dy += MAX_SPLIT_TILE_SIZE) {
dh = (dy + MAX_SPLIT_TILE_SIZE <= y + h) ?
MAX_SPLIT_TILE_SIZE : (y + h - dy);
dw = (w_prev > MAX_SPLIT_TILE_SIZE) ?
MAX_SPLIT_TILE_SIZE : w_prev;
if (!CheckSolidTile(source, x, dy, dw, dh, &colorValue, TRUE))
break;
for (dx = x + dw; dx < x + w_prev;) {
dw = (dx + MAX_SPLIT_TILE_SIZE <= x + w_prev) ?
MAX_SPLIT_TILE_SIZE : (x + w_prev - dx);
if (!CheckSolidTile(source, dx, dy, dw, dh, &colorValue, TRUE))
break;
dx += dw;
}
w_prev = dx - x;
if (w_prev * (dy + dh - y) > w_best * h_best) {
w_best = w_prev;
h_best = dy + dh - y;
}
}
*w_ptr = w_best;
*h_ptr = h_best;
}
void vncEncodeTight::ExtendSolidArea(BYTE *source, int x, int y, int w, int h,
CARD32 colorValue,
int *x_ptr, int *y_ptr,
int *w_ptr, int *h_ptr)
{
int cx, cy;
// Try to extend the area upwards.
for ( cy = *y_ptr - 1;
cy >= y && CheckSolidTile(source, *x_ptr, cy, *w_ptr, 1,
&colorValue, TRUE);
cy-- );
*h_ptr += *y_ptr - (cy + 1);
*y_ptr = cy + 1;
// ... downwards.
for ( cy = *y_ptr + *h_ptr;
cy < y + h && CheckSolidTile(source, *x_ptr, cy, *w_ptr, 1,
&colorValue, TRUE);
cy++ );
*h_ptr += cy - (*y_ptr + *h_ptr);
// ... to the left.
for ( cx = *x_ptr - 1;
cx >= x && CheckSolidTile(source, cx, *y_ptr, 1, *h_ptr,
&colorValue, TRUE);
cx-- );
*w_ptr += *x_ptr - (cx + 1);
*x_ptr = cx + 1;
// ... to the right.
for ( cx = *x_ptr + *w_ptr;
cx < x + w && CheckSolidTile(source, cx, *y_ptr, 1, *h_ptr,
&colorValue, TRUE);
cx++ );
*w_ptr += cx - (*x_ptr + *w_ptr);
}
bool
vncEncodeTight::CheckSolidTile(BYTE *source, int x, int y, int w, int h,
CARD32 *colorPtr, bool needSameColor)
{
switch(m_localformat.bitsPerPixel) {
case 32:
return CheckSolidTile32(source, x, y, w, h, colorPtr, needSameColor);
case 16:
return CheckSolidTile16(source, x, y, w, h, colorPtr, needSameColor);
default:
return CheckSolidTile8(source, x, y, w, h, colorPtr, needSameColor);
}
}
#define DEFINE_CHECK_SOLID_FUNCTION(bpp) \
\
bool \
vncEncodeTight::CheckSolidTile##bpp(BYTE *source, int x, int y, int w, int h, \
CARD32 *colorPtr, bool needSameColor) \
{ \
CARD##bpp *fbptr; \
CARD##bpp colorValue; \
int dx, dy; \
\
fbptr = (CARD##bpp *) \
&source[y * m_bytesPerRow + x * (bpp/8)]; \
\
colorValue = *fbptr; \
if (needSameColor && (CARD32)colorValue != *colorPtr) \
return false; \
\
for (dy = 0; dy < h; dy++) { \
for (dx = 0; dx < w; dx++) { \
if (colorValue != fbptr[dx]) \
return false; \
} \
fbptr = (CARD##bpp *)((BYTE *)fbptr + m_bytesPerRow); \
} \
\
*colorPtr = (CARD32)colorValue; \
return true; \
}
DEFINE_CHECK_SOLID_FUNCTION(8)
DEFINE_CHECK_SOLID_FUNCTION(16)
DEFINE_CHECK_SOLID_FUNCTION(32)
UINT
vncEncodeTight::EncodeRectSimple(BYTE *source, VSocket *outConn, BYTE *dest,
const RECT &rect)
{
const int x = rect.left, y = rect.top;
const int w = rect.right - x, h = rect.bottom - y;
const int maxRectSize = m_conf[m_compresslevel].maxRectSize;
const int maxRectWidth = m_conf[m_compresslevel].maxRectWidth;
int partialSize = 0;
if (w > maxRectWidth || w * h > maxRectSize) {
const int subrectMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
const int subrectMaxHeight = maxRectSize / subrectMaxWidth;
int dx, dy, rw, rh;
for (dy = 0; dy < h; dy += subrectMaxHeight) {
for (dx = 0; dx < w; dx += maxRectWidth) {
rw = (dx + maxRectWidth < w) ? maxRectWidth : w - dx;
rh = (dy + subrectMaxHeight < h) ? subrectMaxHeight : h - dy;
partialSize = EncodeSubrect(source, outConn, dest,
x+dx, y+dy, rw, rh);
if (dy + subrectMaxHeight < h || dx + maxRectWidth < w) {
outConn->SendQueued((char *)dest, partialSize);
}
}
}
} else {
partialSize = EncodeSubrect(source, outConn, dest, x, y, w, h);
}
return partialSize;
}
UINT
vncEncodeTight::EncodeSubrect(BYTE *source, VSocket *outConn, BYTE *dest,
int x, int y, int w, int h)
{
SendTightHeader(x, y, w, h);
RECT r;
r.left = x; r.top = y;
r.right = x + w; r.bottom = y + h;
Translate(source, m_buffer, r);
m_paletteMaxColors = w * h / m_conf[m_compresslevel].idxMaxColorsDivisor;
if ( m_paletteMaxColors < 2 &&
w * h >= m_conf[m_compresslevel].monoMinRectSize ) {
m_paletteMaxColors = 2;
}
switch (m_remoteformat.bitsPerPixel) {
case 8:
FillPalette8(w * h);
break;
case 16:
FillPalette16(w * h);
break;
default:
FillPalette32(w * h);
}
int encDataSize;
switch (m_paletteNumColors) {
case 0:
// Truecolor image
if (DetectSmoothImage(w, h)) {
if (m_qualitylevel != -1) {
encDataSize = SendJpegRect(dest, w, h,
m_conf[m_qualitylevel].jpegQuality);
} else {
encDataSize = SendGradientRect(dest, w, h);
}
} else {
encDataSize = SendFullColorRect(dest, w, h);
}
break;
case 1:
// Solid rectangle
encDataSize = SendSolidRect(dest);
break;
case 2:
// Two-color rectangle
encDataSize = SendMonoRect(dest, w, h);
break;
default:
// Up to 256 different colors
if ( m_paletteNumColors > 96 &&
m_qualitylevel != -1 && m_qualitylevel <= 3 &&
DetectSmoothImage(w, h) ) {
encDataSize = SendJpegRect(dest, w, h,
m_conf[m_qualitylevel].jpegQuality);
} else {
encDataSize = SendIndexedRect(dest, w, h);
}
}
if (encDataSize < 0)
return vncEncoder::EncodeRect(source, dest, r, 0, 0);
outConn->SendQueued((char *)m_hdrBuffer, m_hdrBufferBytes);
encodedSize += m_hdrBufferBytes - sz_rfbFramebufferUpdateRectHeader + encDataSize;
transmittedSize += m_hdrBufferBytes + encDataSize;
return encDataSize;
}
void
vncEncodeTight::SendTightHeader(int x, int y, int w, int h)
{
rfbFramebufferUpdateRectHeader rect;
rect.r.x = Swap16IfLE(x - offsetx);
rect.r.y = Swap16IfLE(y - offsety);
rect.r.w = Swap16IfLE(w);
rect.r.h = Swap16IfLE(h);
rect.encoding = Swap32IfLE(rfbEncodingTight);
dataSize += w * h * (m_remoteformat.bitsPerPixel / 8);
rectangleOverhead += sz_rfbFramebufferUpdateRectHeader;
memcpy(m_hdrBuffer, (BYTE *)&rect, sz_rfbFramebufferUpdateRectHeader);
m_hdrBufferBytes = sz_rfbFramebufferUpdateRectHeader;
}
//
// Subencoding implementations.
//
int
vncEncodeTight::SendSolidRect(BYTE *dest)
{
int len;
if (m_usePixelFormat24) {
Pack24(m_buffer, 1);
len = 3;
} else
len = m_remoteformat.bitsPerPixel / 8;
m_hdrBuffer[m_hdrBufferBytes++] = rfbTightFill << 4;
memcpy (dest, m_buffer, len);
return len;
}
int
vncEncodeTight::SendMonoRect(BYTE *dest, int w, int h)
{
const int streamId = 1;
int paletteLen, dataLen;
CARD8 paletteBuf[8];
// Prepare tight encoding header.
dataLen = (w + 7) / 8;
dataLen *= h;
m_hdrBuffer[m_hdrBufferBytes++] = (streamId | rfbTightExplicitFilter) << 4;
m_hdrBuffer[m_hdrBufferBytes++] = rfbTightFilterPalette;
m_hdrBuffer[m_hdrBufferBytes++] = 1;
// Prepare palette, convert image.
switch (m_remoteformat.bitsPerPixel) {
case 32:
EncodeMonoRect32((CARD8 *)m_buffer, w, h);
((CARD32 *)paletteBuf)[0] = m_monoBackground;
((CARD32 *)paletteBuf)[1] = m_monoForeground;
if (m_usePixelFormat24) {
Pack24(paletteBuf, 2);
paletteLen = 6;
} else
paletteLen = 8;
memcpy(&m_hdrBuffer[m_hdrBufferBytes], paletteBuf, paletteLen);
m_hdrBufferBytes += paletteLen;
break;
case 16:
EncodeMonoRect16((CARD8 *)m_buffer, w, h);
((CARD16 *)paletteBuf)[0] = (CARD16)m_monoBackground;
((CARD16 *)paletteBuf)[1] = (CARD16)m_monoForeground;
memcpy(&m_hdrBuffer[m_hdrBufferBytes], paletteBuf, 4);
m_hdrBufferBytes += 4;
break;
default:
EncodeMonoRect8((CARD8 *)m_buffer, w, h);
m_hdrBuffer[m_hdrBufferBytes++] = (BYTE)m_monoBackground;
m_hdrBuffer[m_hdrBufferBytes++] = (BYTE)m_monoForeground;
}
return CompressData(dest, streamId, dataLen,
m_conf[m_compresslevel].monoZlibLevel,
Z_DEFAULT_STRATEGY);
}
int
vncEncodeTight::SendIndexedRect(BYTE *dest, int w, int h)
{
const int streamId = 2;
int i, entryLen;
CARD8 paletteBuf[256*4];
// Prepare tight encoding header.
m_hdrBuffer[m_hdrBufferBytes++] = (streamId | rfbTightExplicitFilter) << 4;
m_hdrBuffer[m_hdrBufferBytes++] = rfbTightFilterPalette;
m_hdrBuffer[m_hdrBufferBytes++] = (BYTE)(m_paletteNumColors - 1);
// Prepare palette, convert image.
switch (m_remoteformat.bitsPerPixel) {
case 32:
EncodeIndexedRect32((CARD8 *)m_buffer, w * h);
for (i = 0; i < m_paletteNumColors; i++) {
((CARD32 *)paletteBuf)[i] =
m_palette.entry[i].listNode->rgb;
}
if (m_usePixelFormat24) {
Pack24(paletteBuf, m_paletteNumColors);
entryLen = 3;
} else
entryLen = 4;
memcpy(&m_hdrBuffer[m_hdrBufferBytes], paletteBuf,
m_paletteNumColors * entryLen);
m_hdrBufferBytes += m_paletteNumColors * entryLen;
break;
case 16:
EncodeIndexedRect16((CARD8 *)m_buffer, w * h);
for (i = 0; i < m_paletteNumColors; i++) {
((CARD16 *)paletteBuf)[i] =
(CARD16)m_palette.entry[i].listNode->rgb;
}
memcpy(&m_hdrBuffer[m_hdrBufferBytes], paletteBuf,
m_paletteNumColors * 2);
m_hdrBufferBytes += m_paletteNumColors * 2;
break;
default:
return -1; // Should never happen.
}
return CompressData(dest, streamId, w * h,
m_conf[m_compresslevel].idxZlibLevel,
Z_DEFAULT_STRATEGY);
}
int
vncEncodeTight::SendFullColorRect(BYTE *dest, int w, int h)
{
const int streamId = 0;
int len;
m_hdrBuffer[m_hdrBufferBytes++] = 0x00;
if (m_usePixelFormat24) {
Pack24(m_buffer, w * h);
len = 3;
} else
len = m_remoteformat.bitsPerPixel / 8;
return CompressData(dest, streamId, w * h * len,
m_conf[m_compresslevel].rawZlibLevel,
Z_DEFAULT_STRATEGY);
}
int
vncEncodeTight::SendGradientRect(BYTE *dest, int w, int h)
{
const int streamId = 3;
int len;
if (m_remoteformat.bitsPerPixel == 8)
return SendFullColorRect(dest, w, h);
if (m_prevRowBuf == NULL)
m_prevRowBuf = new int [2048*3];
m_hdrBuffer[m_hdrBufferBytes++] = (streamId | rfbTightExplicitFilter) << 4;
m_hdrBuffer[m_hdrBufferBytes++] = rfbTightFilterGradient;
if (m_usePixelFormat24) {
FilterGradient24(m_buffer, w, h);
len = 3;
} else if (m_remoteformat.bitsPerPixel == 32) {
FilterGradient32((CARD32 *)m_buffer, w, h);
len = 4;
} else {
FilterGradient16((CARD16 *)m_buffer, w, h);
len = 2;
}
return CompressData(dest, streamId, w * h * len,
m_conf[m_compresslevel].gradientZlibLevel,
Z_FILTERED);
}
In with the modified VNC payload which now supports an in memory breakout of session isolation for systems like Vista/2008/7 when the payload is run from a service in session 0 isolation. git-svn-id: file:///home/svn/framework3/trunk@8890 4d416f70-5f16-0410-b530-b9f4589650da
2010-03-24 00:00:05 +00:00
/*
VOID vncEncodeTight::UpdateZLibDictionary( AGENT_CTX * lpAgentContext )
{
do
{
for( int i = 0; i < 4; i++ )
{
if( !lpAgentContext->dictionaries[i] )
continue;
setdictionary( &m_zsStruct[i], lpAgentContext->dictionaries[i]->bDictBuffer, lpAgentContext->dictionaries[i]->dwDictLength );
}
} while( 0 );
}
VOID vncEncodeTight::DumpZLibDictionary( AGENT_CTX * lpAgentContext )
{
do
{
for( int i = 0; i < 4; i++ )
{
if( !m_zsActive[i] )
continue;
SendZlibDictionary( lpAgentContext, i, &m_zsStruct[i] );
}
} while( 0 );
}
*/
int vncEncodeTight::CompressData(BYTE *dest, int streamId, int dataLen,
Commit the new VNC x86/x64 DLL source code... git-svn-id: file:///home/svn/framework3/trunk@8745 4d416f70-5f16-0410-b530-b9f4589650da
2010-03-08 14:49:45 +00:00
int zlibLevel, int zlibStrategy)
{
if (dataLen < TIGHT_MIN_TO_COMPRESS) {
memcpy(dest, m_buffer, dataLen);
return dataLen;
}
z_streamp pz = &m_zsStruct[streamId];
// Initialize compression stream if needed.
if (!m_zsActive[streamId]) {
pz->zalloc = Z_NULL;
pz->zfree = Z_NULL;
pz->opaque = Z_NULL;
int err = deflateInit2 (pz, zlibLevel, Z_DEFLATED, MAX_WBITS,
MAX_MEM_LEVEL, zlibStrategy);
if (err != Z_OK) {
return -1;
}
m_zsActive[streamId] = true;
m_zsLevel[streamId] = zlibLevel;
}
int outBufferSize = dataLen + dataLen / 100 + 16;
// Prepare buffer pointers.
pz->next_in = (Bytef *)m_buffer;
pz->avail_in = dataLen;
pz->next_out = (Bytef *)dest;
pz->avail_out = outBufferSize;
// Change compression parameters if needed.
if (zlibLevel != m_zsLevel[streamId]) {
int err = deflateParams (pz, zlibLevel, zlibStrategy);
if (err != Z_OK) {
return -1;
}
m_zsLevel[streamId] = zlibLevel;
}
// Actual compression.
if ( deflate (pz, Z_SYNC_FLUSH) != Z_OK ||
pz->avail_in != 0 || pz->avail_out == 0 ) {
return -1;
}
return SendCompressedData(outBufferSize - pz->avail_out);
}
int
vncEncodeTight::SendCompressedData(size_t compressedLen)
{
// Prepare compressed data size for sending.
m_hdrBuffer[m_hdrBufferBytes++] = compressedLen & 0x7F;
if (compressedLen > 0x7F) {
m_hdrBuffer[m_hdrBufferBytes-1] |= 0x80;
m_hdrBuffer[m_hdrBufferBytes++] = compressedLen >> 7 & 0x7F;
if (compressedLen > 0x3FFF) {
m_hdrBuffer[m_hdrBufferBytes-1] |= 0x80;
m_hdrBuffer[m_hdrBufferBytes++] = compressedLen >> 14 & 0xFF;
}
}
return (int)compressedLen;
}
void
vncEncodeTight::FillPalette8(int count)
{
CARD8 *data = (CARD8 *)m_buffer;
CARD8 c0, c1;
int i, n0, n1;
m_paletteNumColors = 0;
c0 = data[0];
for (i = 1; i < count && data[i] == c0; i++);
if (i == count) {
m_paletteNumColors = 1;
return; // Solid rectangle
}
if (m_paletteMaxColors < 2)
return;
n0 = i;
c1 = data[i];
n1 = 0;
for (i++; i < count; i++) {
if (data[i] == c0) {
n0++;
} else if (data[i] == c1) {
n1++;
} else
break;
}
if (i == count) {
if (n0 > n1) {
m_monoBackground = (CARD32)c0;
m_monoForeground = (CARD32)c1;
} else {
m_monoBackground = (CARD32)c1;
m_monoForeground = (CARD32)c0;
}
m_paletteNumColors = 2; // Two colors
}
}
#define DEFINE_FILL_PALETTE_FUNCTION(bpp) \
\
void \
vncEncodeTight::FillPalette##bpp(int count) \
{ \
CARD##bpp *data = (CARD##bpp *)m_buffer; \
CARD##bpp c0, c1, ci; \
int i, n0, n1, ni; \
\
c0 = data[0]; \
for (i = 1; i < count && data[i] == c0; i++); \
if (i >= count) { \
m_paletteNumColors = 1; /* Solid rectangle */ \
return; \
} \
\
if (m_paletteMaxColors < 2) { \
m_paletteNumColors = 0; /* Full-color format preferred */ \
return; \
} \
\
n0 = i; \
c1 = data[i]; \
n1 = 0; \
for (i++; i < count; i++) { \
ci = data[i]; \
if (ci == c0) { \
n0++; \
} else if (ci == c1) { \
n1++; \
} else \
break; \
} \
if (i >= count) { \
if (n0 > n1) { \
m_monoBackground = (CARD32)c0; \
m_monoForeground = (CARD32)c1; \
} else { \
m_monoBackground = (CARD32)c1; \
m_monoForeground = (CARD32)c0; \
} \
m_paletteNumColors = 2; /* Two colors */ \
return; \
} \
\
PaletteReset(); \
PaletteInsert (c0, (CARD32)n0, bpp); \
PaletteInsert (c1, (CARD32)n1, bpp); \
\
ni = 1; \
for (i++; i < count; i++) { \
if (data[i] == ci) { \
ni++; \
} else { \
if (!PaletteInsert (ci, (CARD32)ni, bpp)) \
return; \
ci = data[i]; \
ni = 1; \
} \
} \
PaletteInsert (ci, (CARD32)ni, bpp); \
}
DEFINE_FILL_PALETTE_FUNCTION(16)
DEFINE_FILL_PALETTE_FUNCTION(32)
//
// Functions to operate with palette structures.
//
#define HASH_FUNC16(rgb) ((int)(((rgb >> 8) + rgb) & 0xFF))
#define HASH_FUNC32(rgb) ((int)(((rgb >> 16) + (rgb >> 8)) & 0xFF))
void
vncEncodeTight::PaletteReset(void)
{
m_paletteNumColors = 0;
memset(m_palette.hash, 0, 256 * sizeof(COLOR_LIST *));
}
int
vncEncodeTight::PaletteInsert(CARD32 rgb, int numPixels, int bpp)
{
COLOR_LIST *pnode;
COLOR_LIST *prev_pnode = NULL;
int hash_key, idx, new_idx, count;
hash_key = (bpp == 16) ? HASH_FUNC16(rgb) : HASH_FUNC32(rgb);
pnode = m_palette.hash[hash_key];
while (pnode != NULL) {
if (pnode->rgb == rgb) {
// Such palette entry already exists.
new_idx = idx = pnode->idx;
count = m_palette.entry[idx].numPixels + numPixels;
if (new_idx && m_palette.entry[new_idx-1].numPixels < count) {
do {
m_palette.entry[new_idx] = m_palette.entry[new_idx-1];
m_palette.entry[new_idx].listNode->idx = new_idx;
new_idx--;
}
while (new_idx &&
m_palette.entry[new_idx-1].numPixels < count);
m_palette.entry[new_idx].listNode = pnode;
pnode->idx = new_idx;
}
m_palette.entry[new_idx].numPixels = count;
return m_paletteNumColors;
}
prev_pnode = pnode;
pnode = pnode->next;
}
// Check if palette is full.
if ( m_paletteNumColors == 256 ||
m_paletteNumColors == m_paletteMaxColors ) {
m_paletteNumColors = 0;
return 0;
}
// Move palette entries with lesser pixel counts.
for ( idx = m_paletteNumColors;
idx > 0 && m_palette.entry[idx-1].numPixels < numPixels;
idx-- ) {
m_palette.entry[idx] = m_palette.entry[idx-1];
m_palette.entry[idx].listNode->idx = idx;
}
// Add new palette entry into the freed slot.
pnode = &m_palette.list[m_paletteNumColors];
if (prev_pnode != NULL) {
prev_pnode->next = pnode;
} else {
m_palette.hash[hash_key] = pnode;
}
pnode->next = NULL;
pnode->idx = idx;
pnode->rgb = rgb;
m_palette.entry[idx].listNode = pnode;
m_palette.entry[idx].numPixels = numPixels;
return (++m_paletteNumColors);
}
//
// Converting 32-bit color samples into 24-bit colors.
// Should be called only when redMax, greenMax and blueMax are 255.
// Color components assumed to be byte-aligned.
//
void
vncEncodeTight::Pack24(BYTE *buf, int count)
{
CARD32 *buf32;
CARD32 pix;
int r_shift, g_shift, b_shift;
buf32 = (CARD32 *)buf;
if (!m_localformat.bigEndian == !m_remoteformat.bigEndian) {
r_shift = m_remoteformat.redShift;
g_shift = m_remoteformat.greenShift;
b_shift = m_remoteformat.blueShift;
} else {
r_shift = 24 - m_remoteformat.redShift;
g_shift = 24 - m_remoteformat.greenShift;
b_shift = 24 - m_remoteformat.blueShift;
}
while (count--) {
pix = *buf32++;
*buf++ = (char)(pix >> r_shift);
*buf++ = (char)(pix >> g_shift);
*buf++ = (char)(pix >> b_shift);
}
}
//
// Converting truecolor samples into palette indices.
//
#define DEFINE_IDX_ENCODE_FUNCTION(bpp) \
\
void \
vncEncodeTight::EncodeIndexedRect##bpp(BYTE *buf, int count) \
{ \
COLOR_LIST *pnode; \
CARD##bpp *src; \
CARD##bpp rgb; \
int rep = 0; \
\
src = (CARD##bpp *) buf; \
\
while (count--) { \
rgb = *src++; \
while (count && *src == rgb) { \
rep++, src++, count--; \
} \
pnode = m_palette.hash[HASH_FUNC##bpp(rgb)]; \
while (pnode != NULL) { \
if ((CARD##bpp)pnode->rgb == rgb) { \
*buf++ = (CARD8)pnode->idx; \
while (rep) { \
*buf++ = (CARD8)pnode->idx; \
rep--; \
} \
break; \
} \
pnode = pnode->next; \
} \
} \
}
DEFINE_IDX_ENCODE_FUNCTION(16)
DEFINE_IDX_ENCODE_FUNCTION(32)
#define DEFINE_MONO_ENCODE_FUNCTION(bpp) \
\
void \
vncEncodeTight::EncodeMonoRect##bpp(BYTE *buf, int w, int h) \
{ \
CARD##bpp *ptr; \
CARD##bpp bg; \
unsigned int value, mask; \
int aligned_width; \
int x, y, bg_bits; \
\
ptr = (CARD##bpp *) buf; \
bg = (CARD##bpp) m_monoBackground; \
aligned_width = w - w % 8; \
\
for (y = 0; y < h; y++) { \
for (x = 0; x < aligned_width; x += 8) { \
for (bg_bits = 0; bg_bits < 8; bg_bits++) { \
if (*ptr++ != bg) \
break; \
} \
if (bg_bits == 8) { \
*buf++ = 0; \
continue; \
} \
mask = 0x80 >> bg_bits; \
value = mask; \
for (bg_bits++; bg_bits < 8; bg_bits++) { \
mask >>= 1; \
if (*ptr++ != bg) { \
value |= mask; \
} \
} \
*buf++ = (CARD8)value; \
} \
\
mask = 0x80; \
value = 0; \
if (x >= w) \
continue; \
\
for (; x < w; x++) { \
if (*ptr++ != bg) { \
value |= mask; \
} \
mask >>= 1; \
} \
*buf++ = (CARD8)value; \
} \
}
DEFINE_MONO_ENCODE_FUNCTION(8)
DEFINE_MONO_ENCODE_FUNCTION(16)
DEFINE_MONO_ENCODE_FUNCTION(32)
//
// ``Gradient'' filter for 24-bit color samples.
// Should be called only when redMax, greenMax and blueMax are 255.
// Color components assumed to be byte-aligned.
//
void
vncEncodeTight::FilterGradient24(BYTE *buf, int w, int h)
{
CARD32 *buf32;
CARD32 pix32;
int *prevRowPtr;
int shiftBits[3];
int pixHere[3], pixUpper[3], pixLeft[3], pixUpperLeft[3];
int prediction;
int x, y, c;
buf32 = (CARD32 *)buf;
memset (m_prevRowBuf, 0, w * 3 * sizeof(int));
if (!m_localformat.bigEndian == !m_remoteformat.bigEndian) {
shiftBits[0] = m_remoteformat.redShift;
shiftBits[1] = m_remoteformat.greenShift;
shiftBits[2] = m_remoteformat.blueShift;
} else {
shiftBits[0] = 24 - m_remoteformat.redShift;
shiftBits[1] = 24 - m_remoteformat.greenShift;
shiftBits[2] = 24 - m_remoteformat.blueShift;
}
for (y = 0; y < h; y++) {
for (c = 0; c < 3; c++) {
pixUpper[c] = 0;
pixHere[c] = 0;
}
prevRowPtr = m_prevRowBuf;
for (x = 0; x < w; x++) {
pix32 = *buf32++;
for (c = 0; c < 3; c++) {
pixUpperLeft[c] = pixUpper[c];
pixLeft[c] = pixHere[c];
pixUpper[c] = *prevRowPtr;
pixHere[c] = (int)(pix32 >> shiftBits[c] & 0xFF);
*prevRowPtr++ = pixHere[c];
prediction = pixLeft[c] + pixUpper[c] - pixUpperLeft[c];
if (prediction < 0) {
prediction = 0;
} else if (prediction > 0xFF) {
prediction = 0xFF;
}
*buf++ = (BYTE)(pixHere[c] - prediction);
}
}
}
}
//
// ``Gradient'' filter for other color depths.
//
#define DEFINE_GRADIENT_FILTER_FUNCTION(bpp) \
\
void \
vncEncodeTight::FilterGradient##bpp(CARD##bpp *buf, int w, int h) \
{ \
CARD##bpp pix, diff; \
bool endianMismatch; \
int *prevRowPtr; \
int maxColor[3], shiftBits[3]; \
int pixHere[3], pixUpper[3], pixLeft[3], pixUpperLeft[3]; \
int prediction; \
int x, y, c; \
\
memset (m_prevRowBuf, 0, w * 3 * sizeof(int)); \
\
endianMismatch = (!m_localformat.bigEndian != !m_remoteformat.bigEndian); \
\
maxColor[0] = m_remoteformat.redMax; \
maxColor[1] = m_remoteformat.greenMax; \
maxColor[2] = m_remoteformat.blueMax; \
shiftBits[0] = m_remoteformat.redShift; \
shiftBits[1] = m_remoteformat.greenShift; \
shiftBits[2] = m_remoteformat.blueShift; \
\
for (y = 0; y < h; y++) { \
for (c = 0; c < 3; c++) { \
pixUpper[c] = 0; \
pixHere[c] = 0; \
} \
prevRowPtr = m_prevRowBuf; \
for (x = 0; x < w; x++) { \
pix = *buf; \
if (endianMismatch) { \
pix = Swap##bpp(pix); \
} \
diff = 0; \
for (c = 0; c < 3; c++) { \
pixUpperLeft[c] = pixUpper[c]; \
pixLeft[c] = pixHere[c]; \
pixUpper[c] = *prevRowPtr; \
pixHere[c] = (int)(pix >> shiftBits[c] & maxColor[c]); \
*prevRowPtr++ = pixHere[c]; \
\
prediction = pixLeft[c] + pixUpper[c] - pixUpperLeft[c]; \
if (prediction < 0) { \
prediction = 0; \
} else if (prediction > maxColor[c]) { \
prediction = maxColor[c]; \
} \
diff |= ((pixHere[c] - prediction) & maxColor[c]) \
<< shiftBits[c]; \
} \
if (endianMismatch) { \
diff = Swap##bpp(diff); \
} \
*buf++ = diff; \
} \
} \
}
DEFINE_GRADIENT_FILTER_FUNCTION(16)
DEFINE_GRADIENT_FILTER_FUNCTION(32)
//
// Code to guess if given rectangle is suitable for smooth image
// compression (by applying "gradient" filter or JPEG coder).
//
#define JPEG_MIN_RECT_SIZE 4096
#define DETECT_SUBROW_WIDTH 7
#define DETECT_MIN_WIDTH 8
#define DETECT_MIN_HEIGHT 8
int
vncEncodeTight::DetectSmoothImage (int w, int h)
{
if ( m_localformat.bitsPerPixel == 8 || m_remoteformat.bitsPerPixel == 8 ||
w < DETECT_MIN_WIDTH || h < DETECT_MIN_HEIGHT ) {
return 0;
}
if (m_qualitylevel != -1) {
if (w * h < JPEG_MIN_RECT_SIZE) {
return 0;
}
} else {
if (w * h < m_conf[m_compresslevel].gradientMinRectSize) {
return 0;
}
}
unsigned long avgError;
if (m_remoteformat.bitsPerPixel == 32) {
if (m_usePixelFormat24) {
avgError = DetectSmoothImage24(w, h);
if (m_qualitylevel != -1) {
return (avgError < m_conf[m_qualitylevel].jpegThreshold24);
}
return (avgError < m_conf[m_compresslevel].gradientThreshold24);
} else {
avgError = DetectSmoothImage32(w, h);
}
} else {
avgError = DetectSmoothImage16(w, h);
}
if (m_qualitylevel != -1) {
return (avgError < m_conf[m_qualitylevel].jpegThreshold);
}
return (avgError < m_conf[m_compresslevel].gradientThreshold);
}
unsigned long
vncEncodeTight::DetectSmoothImage24 (int w, int h)
{
int diffStat[256];
int pixelCount = 0;
int pix, left[3];
unsigned long avgError;
// If client is big-endian, color samples begin from the second
// byte (offset 1) of a 32-bit pixel value.
int off = (m_remoteformat.bigEndian != 0);
memset(diffStat, 0, 256*sizeof(int));
int y = 0, x = 0;
int d, dx, c;
while (y < h && x < w) {
for (d = 0; d < h - y && d < w - x - DETECT_SUBROW_WIDTH; d++) {
for (c = 0; c < 3; c++) {
left[c] = (int)m_buffer[((y+d)*w+x+d)*4+off+c] & 0xFF;
}
for (dx = 1; dx <= DETECT_SUBROW_WIDTH; dx++) {
for (c = 0; c < 3; c++) {
pix = (int)m_buffer[((y+d)*w+x+d+dx)*4+off+c] & 0xFF;
diffStat[abs(pix - left[c])]++;
left[c] = pix;
}
pixelCount++;
}
}
if (w > h) {
x += h;
y = 0;
} else {
x = 0;
y += w;
}
}
if (diffStat[0] * 33 / pixelCount >= 95)
return 0;
avgError = 0;
for (c = 1; c < 8; c++) {
avgError += (unsigned long)diffStat[c] * (unsigned long)(c * c);
if (diffStat[c] == 0 || diffStat[c] > diffStat[c-1] * 2)
return 0;
}
for (; c < 256; c++) {
avgError += (unsigned long)diffStat[c] * (unsigned long)(c * c);
}
avgError /= (pixelCount * 3 - diffStat[0]);
return avgError;
}
#define DEFINE_DETECT_FUNCTION(bpp) \
\
unsigned long \
vncEncodeTight::DetectSmoothImage##bpp (int w, int h) \
{ \
bool endianMismatch; \
CARD##bpp pix; \
int maxColor[3], shiftBits[3]; \
int x, y, d, dx, c; \
int diffStat[256]; \
int pixelCount = 0; \
int sample, sum, left[3]; \
unsigned long avgError; \
\
endianMismatch = (!m_localformat.bigEndian != !m_remoteformat.bigEndian); \
\
maxColor[0] = m_remoteformat.redMax; \
maxColor[1] = m_remoteformat.greenMax; \
maxColor[2] = m_remoteformat.blueMax; \
shiftBits[0] = m_remoteformat.redShift; \
shiftBits[1] = m_remoteformat.greenShift; \
shiftBits[2] = m_remoteformat.blueShift; \
\
memset(diffStat, 0, 256*sizeof(int)); \
\
y = 0, x = 0; \
while (y < h && x < w) { \
for (d = 0; d < h - y && d < w - x - DETECT_SUBROW_WIDTH; d++) { \
pix = ((CARD##bpp *)m_buffer)[(y+d)*w+x+d]; \
if (endianMismatch) { \
pix = Swap##bpp(pix); \
} \
for (c = 0; c < 3; c++) { \
left[c] = (int)(pix >> shiftBits[c] & maxColor[c]); \
} \
for (dx = 1; dx <= DETECT_SUBROW_WIDTH; dx++) { \
pix = ((CARD##bpp *)m_buffer)[(y+d)*w+x+d+dx]; \
if (endianMismatch) { \
pix = Swap##bpp(pix); \
} \
sum = 0; \
for (c = 0; c < 3; c++) { \
sample = (int)(pix >> shiftBits[c] & maxColor[c]); \
sum += abs(sample - left[c]); \
left[c] = sample; \
} \
if (sum > 255) \
sum = 255; \
diffStat[sum]++; \
pixelCount++; \
} \
} \
if (w > h) { \
x += h; \
y = 0; \
} else { \
x = 0; \
y += w; \
} \
} \
\
if ((diffStat[0] + diffStat[1]) * 100 / pixelCount >= 90) \
return 0; \
\
avgError = 0; \
for (c = 1; c < 8; c++) { \
avgError += (unsigned long)diffStat[c] * (unsigned long)(c * c); \
if (diffStat[c] == 0 || diffStat[c] > diffStat[c-1] * 2) \
return 0; \
} \
for (; c < 256; c++) { \
avgError += (unsigned long)diffStat[c] * (unsigned long)(c * c); \
} \
avgError /= (pixelCount - diffStat[0]); \
\
return avgError; \
}
DEFINE_DETECT_FUNCTION(16)
DEFINE_DETECT_FUNCTION(32)
//
// JPEG compression stuff.
//
static bool jpegError;
static size_t jpegDstDataLen;
static void JpegSetDstManager(j_compress_ptr cinfo, JOCTET *buf, size_t buflen);
int
vncEncodeTight::SendJpegRect(BYTE *dst, int w, int h, int quality)
{
struct jpeg_compress_struct cinfo;
struct jpeg_error_mgr jerr;
if (m_localformat.bitsPerPixel == 8)
return SendFullColorRect(dst, w, h);
BYTE *srcBuf = new byte[w * 3];
JSAMPROW rowPointer[1];
rowPointer[0] = (JSAMPROW)srcBuf;
cinfo.err = jpeg_std_error(&jerr);
jpeg_create_compress(&cinfo);
cinfo.image_width = w;
cinfo.image_height = h;
cinfo.input_components = 3;
cinfo.in_color_space = JCS_RGB;
jpeg_set_defaults(&cinfo);
jpeg_set_quality(&cinfo, quality, TRUE);
JpegSetDstManager (&cinfo, dst, w * h * (m_localformat.bitsPerPixel / 8));
jpeg_start_compress(&cinfo, TRUE);
for (int dy = 0; dy < h; dy++) {
PrepareRowForJpeg(srcBuf, dy, w);
jpeg_write_scanlines(&cinfo, rowPointer, 1);
if (jpegError)
break;
}
if (!jpegError)
jpeg_finish_compress(&cinfo);
jpeg_destroy_compress(&cinfo);
delete[] srcBuf;
if (jpegError)
return SendFullColorRect(dst, w, h);
m_hdrBuffer[m_hdrBufferBytes++] = rfbTightJpeg << 4;
return SendCompressedData(jpegDstDataLen);
}
void
vncEncodeTight::PrepareRowForJpeg(BYTE *dst, int y, int w)
{
if (m_remoteformat.bitsPerPixel == 32) {
CARD32 *src = (CARD32 *)&m_buffer[y * w * sizeof(CARD32)];
if (m_usePixelFormat24) {
PrepareRowForJpeg24(dst, src, w);
} else {
PrepareRowForJpeg32(dst, src, w);
}
} else {
// 16 bpp assumed.
CARD16 *src = (CARD16 *)&m_buffer[y * w * sizeof(CARD16)];
PrepareRowForJpeg16(dst, src, w);
}
}
void
vncEncodeTight::PrepareRowForJpeg24(BYTE *dst, CARD32 *src, int count)
{
int r_shift, g_shift, b_shift;
if (!m_localformat.bigEndian == !m_remoteformat.bigEndian) {
r_shift = m_remoteformat.redShift;
g_shift = m_remoteformat.greenShift;
b_shift = m_remoteformat.blueShift;
} else {
r_shift = 24 - m_remoteformat.redShift;
g_shift = 24 - m_remoteformat.greenShift;
b_shift = 24 - m_remoteformat.blueShift;
}
CARD32 pix;
while (count--) {
pix = *src++;
*dst++ = (BYTE)(pix >> r_shift);
*dst++ = (BYTE)(pix >> g_shift);
*dst++ = (BYTE)(pix >> b_shift);
}
}
#define DEFINE_JPEG_GET_ROW_FUNCTION(bpp) \
\
void \
vncEncodeTight::PrepareRowForJpeg##bpp(BYTE *dst, CARD##bpp *src, int count)\
{ \
bool endianMismatch = \
(!m_localformat.bigEndian != !m_remoteformat.bigEndian); \
\
int r_shift = m_remoteformat.redShift; \
int g_shift = m_remoteformat.greenShift; \
int b_shift = m_remoteformat.blueShift; \
int r_max = m_remoteformat.redMax; \
int g_max = m_remoteformat.greenMax; \
int b_max = m_remoteformat.blueMax; \
\
CARD##bpp pix; \
while (count--) { \
pix = *src++; \
if (endianMismatch) { \
pix = Swap##bpp(pix); \
} \
*dst++ = (BYTE)((pix >> r_shift & r_max) * 255 / r_max); \
*dst++ = (BYTE)((pix >> g_shift & g_max) * 255 / g_max); \
*dst++ = (BYTE)((pix >> b_shift & b_max) * 255 / b_max); \
} \
}
DEFINE_JPEG_GET_ROW_FUNCTION(16)
DEFINE_JPEG_GET_ROW_FUNCTION(32)
/*
* Destination manager implementation for JPEG library.
*/
static struct jpeg_destination_mgr jpegDstManager;
static JOCTET *jpegDstBuffer;
static size_t jpegDstBufferLen;
static void JpegInitDestination(j_compress_ptr cinfo);
static boolean JpegEmptyOutputBuffer(j_compress_ptr cinfo);
static void JpegTermDestination(j_compress_ptr cinfo);
static void
JpegInitDestination(j_compress_ptr cinfo)
{
jpegError = false;
jpegDstManager.next_output_byte = jpegDstBuffer;
jpegDstManager.free_in_buffer = jpegDstBufferLen;
}
static boolean
JpegEmptyOutputBuffer(j_compress_ptr cinfo)
{
jpegError = true;
jpegDstManager.next_output_byte = jpegDstBuffer;
jpegDstManager.free_in_buffer = jpegDstBufferLen;
return TRUE;
}
static void
JpegTermDestination(j_compress_ptr cinfo)
{
jpegDstDataLen = jpegDstBufferLen - jpegDstManager.free_in_buffer;
}
static void
JpegSetDstManager(j_compress_ptr cinfo, JOCTET *buf, size_t buflen)
{
jpegDstBuffer = buf;
jpegDstBufferLen = buflen;
jpegDstManager.init_destination = JpegInitDestination;
jpegDstManager.empty_output_buffer = JpegEmptyOutputBuffer;
jpegDstManager.term_destination = JpegTermDestination;
cinfo->dest = &jpegDstManager;
}