//  Copyright (C) 2000 Constantin Kaplinsky. All Rights Reserved.
//  Copyright (C) 2000 Tridia Corporation. All Rights Reserved.
//  Copyright (C) 1999 AT&T Laboratories Cambridge. All Rights Reserved.
//
//  This file is part of the VNC system.
//
//  The VNC system is free software; you can redistribute it and/or modify
//  it under the terms of the GNU General Public License as published by
//  the Free Software Foundation; either version 2 of the License, or
//  (at your option) any later version.
//
//  This program is distributed in the hope that it will be useful,
//  but WITHOUT ANY WARRANTY; without even the implied warranty of
//  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
//  GNU General Public License for more details.
//
//  You should have received a copy of the GNU General Public License
//  along with this program; if not, write to the Free Software
//  Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307,
//  USA.
//
// TightVNC distribution homepage on the Web: http://www.tightvnc.com/
//
// If the source code for the VNC system is not available from the place 
// whence you received this file, check http://www.uk.research.att.com/vnc or contact
// the authors on vnc@uk.research.att.com for information on obtaining it.


// vncEncodeTight

// This file implements the vncEncoder-derived vncEncodeTight class.
// This class overrides some vncEncoder functions to produce a bitmap
// to Tight encoder. Tight is much more efficient than RAW format on
// most screen data and usually 2..10 times as efficient as hextile.
// It's also more efficient than Zlib encoding in most cases.
// But note that tight compression may use more CPU time on the server.
// However, over slower (128kbps or less) connections, the reduction
// in data transmitted usually outweighs the extra latency added
// while the server CPU performs the compression algorithms.

#include "vncEncodeTight.h"

// Compression level stuff. The following array contains various
// encoder parameters for each of 10 compression levels (0..9).
// Last three parameters correspond to JPEG quality levels (0..9).
//
// NOTE: m_conf[9].maxRectSize should be >= m_conf[i].maxRectSize,
// where i in [0..8]. RequiredBuffSize() method depends on this.

const TIGHT_CONF vncEncodeTight::m_conf[10] = {
	{   512,   32,   6, 65536, 0, 0, 0, 0,   0,   0,   4,  5, 10000, 23000 },
	{  2048,  128,   6, 65536, 1, 1, 1, 0,   0,   0,   8, 10,  8000, 18000 },
	{  6144,  256,   8, 65536, 3, 3, 2, 0,   0,   0,  24, 15,  6500, 15000 },
	{ 10240, 1024,  12, 65536, 5, 5, 3, 0,   0,   0,  32, 25,  5000, 12000 },
	{ 16384, 2048,  12, 65536, 6, 6, 4, 0,   0,   0,  32, 37,  4000, 10000 },
	{ 32768, 2048,  12,  4096, 7, 7, 5, 4, 150, 380,  32, 50,  3000,  8000 },
	{ 65536, 2048,  16,  4096, 7, 7, 6, 4, 170, 420,  48, 60,  2000,  5000 },
	{ 65536, 2048,  16,  4096, 8, 8, 7, 5, 180, 450,  64, 70,  1000,  2500 },
	{ 65536, 2048,  32,  8192, 9, 9, 8, 6, 190, 475,  64, 75,   500,  1200 },
	{ 65536, 2048,  32,  8192, 9, 9, 9, 6, 200, 500,  96, 80,   200,   500 }
};

vncEncodeTight::vncEncodeTight()
{
	m_buffer = NULL;
	m_bufflen = 0;

	m_hdrBuffer = new BYTE [sz_rfbFramebufferUpdateRectHeader + 8 + 256*4];
	m_prevRowBuf = NULL;

	for (int i = 0; i < 4; i++)
		m_zsActive[i] = false;
}

vncEncodeTight::~vncEncodeTight()
{
	if (m_buffer != NULL) {
		delete[] m_buffer;
		m_buffer = NULL;
	}

	delete[] m_hdrBuffer;

	for (int i = 0; i < 4; i++) {
		if (m_zsActive[i])
			deflateEnd(&m_zsStruct[i]);
		m_zsActive[i] = false;
	}
}

void
vncEncodeTight::Init()
{
	vncEncoder::Init();
}

/*****************************************************************************
 *
 * Routines to implement Tight Encoding.
 *
 */

UINT
vncEncodeTight::RequiredBuffSize(UINT width, UINT height)
{
	// FIXME: Use actual compression level instead of 9?
	int result = m_conf[9].maxRectSize * (m_remoteformat.bitsPerPixel / 8);
	result += result / 100 + 16;

	return result;
}

UINT
vncEncodeTight::NumCodedRects(RECT &rect)
{
	const int w = rect.right - rect.left;
	const int h = rect.bottom - rect.top;

	// No matter how many rectangles we will send if LastRect markers
	// are used to terminate rectangle stream.
	if (m_use_lastrect && w * h >= MIN_SPLIT_RECT_SIZE) {
		return 0;
	}

	const int maxRectSize = m_conf[m_compresslevel].maxRectSize;
	const int maxRectWidth = m_conf[m_compresslevel].maxRectWidth;

	if (w > maxRectWidth || w * h > maxRectSize) {
		const int subrectMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
		const int subrectMaxHeight = maxRectSize / subrectMaxWidth;
		return (((w - 1) / maxRectWidth + 1) *
				((h - 1) / subrectMaxHeight + 1));
	} else {
		return 1;
	}
}

UINT
vncEncodeTight::EncodeRect(BYTE *source, VSocket *outConn, BYTE *dest,
						   const RECT &rect, int offx, int offy)
{
	int x = rect.left, y = rect.top;
	int w = rect.right - x, h = rect.bottom - y;
	offsetx = offx;
	offsety = offy;

	const int maxRectSize = m_conf[m_compresslevel].maxRectSize;
	const int rawDataSize = maxRectSize * (m_remoteformat.bitsPerPixel / 8);

	if (m_bufflen < rawDataSize) {
		if (m_buffer != NULL)
			delete [] m_buffer;

		m_buffer = new BYTE [rawDataSize+1];
		if (m_buffer == NULL)
			return vncEncoder::EncodeRect(source, dest, rect, offsetx, offsety);

		m_bufflen = rawDataSize;
	}

	if ( m_remoteformat.depth == 24 && m_remoteformat.redMax == 0xFF &&
		 m_remoteformat.greenMax == 0xFF && m_remoteformat.blueMax == 0xFF ) {
		m_usePixelFormat24 = true;
	} else {
		m_usePixelFormat24 = false;
	}

	if (!m_use_lastrect || w * h < MIN_SPLIT_RECT_SIZE)
		return EncodeRectSimple(source, outConn, dest, rect);

	// Calculate maximum number of rows in one non-solid rectangle.

	int nMaxRows;
	{
		int maxRectSize = m_conf[m_compresslevel].maxRectSize;
		int maxRectWidth = m_conf[m_compresslevel].maxRectWidth;
		int nMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
		nMaxRows = maxRectSize / nMaxWidth;
	}

	// Try to find large solid-color areas and send them separately.

	CARD32 colorValue;
	int x_best, y_best, w_best, h_best;
	int dx, dy, dw, dh;

	for (dy = y; dy < y + h; dy += MAX_SPLIT_TILE_SIZE) {

		// If a rectangle becomes too large, send its upper part now.

		if (dy - y >= nMaxRows) {
			RECT upperRect;
			SetRect(&upperRect, x, y, x + w, y + nMaxRows);

			int size = EncodeRectSimple(source, outConn, dest, upperRect);
			outConn->SendQueued((char *)dest, size);

			y += nMaxRows;
			h -= nMaxRows;
		}

		dh = (dy + MAX_SPLIT_TILE_SIZE <= y + h) ?
			MAX_SPLIT_TILE_SIZE : (y + h - dy);

		for (dx = x; dx < x + w; dx += MAX_SPLIT_TILE_SIZE) {

			dw = (dx + MAX_SPLIT_TILE_SIZE <= x + w) ?
				MAX_SPLIT_TILE_SIZE : (x + w - dx);

			if (CheckSolidTile(source, dx, dy, dw, dh, &colorValue, FALSE)) {

				// Get dimensions of solid-color area.

				FindBestSolidArea(source, dx, dy, w - (dx - x), h - (dy - y),
								  colorValue, &w_best, &h_best);

				// Make sure a solid rectangle is large enough
				// (or the whole rectangle is of the same color).

				if ( w_best * h_best != w * h &&
					 w_best * h_best < MIN_SOLID_SUBRECT_SIZE )
					continue;

				// Try to extend solid rectangle to maximum size.

				x_best = dx; y_best = dy;
				ExtendSolidArea(source, x, y, w, h, colorValue,
								&x_best, &y_best, &w_best, &h_best);

				// Compute dimensions of surrounding rectangles.

				RECT rects[4];
				SetRect(&rects[0],
						x, y, x + w, y_best);
				SetRect(&rects[1],
						x, y_best, x_best, y_best + h_best);
				SetRect(&rects[2],
						x_best + w_best, y_best, x + w, y_best + h_best);
				SetRect(&rects[3],
						x, y_best + h_best, x + w, y + h);

				// Send solid-color area and surrounding rectangles.

				for (int i = 0; i < 4; i++) {
					if (i == 2) {
						RECT onePixel;
						SetRect(&onePixel,
								x_best, y_best, x_best + 1, y_best + 1);
						Translate(source, m_buffer, onePixel);

						SendTightHeader(x_best, y_best, w_best, h_best);
						int size = SendSolidRect(dest);

						outConn->SendQueued((char *)m_hdrBuffer, m_hdrBufferBytes);
						outConn->SendQueued((char *)dest, size);
						encodedSize += (m_hdrBufferBytes + size -
										sz_rfbFramebufferUpdateRectHeader);
						transmittedSize += (m_hdrBufferBytes + size);
					}
					if ( rects[i].left == rects[i].right ||
						 rects[i].top  == rects[i].bottom ) {
						continue;
					}
					int size = EncodeRect(source, outConn, dest, rects[i], offsetx, offsety);
					outConn->SendQueued((char *)dest, size);
				}

				// Return after all recursive calls done (0 == data sent).

				return 0;
			}

		}

	}

	// No suitable solid-color rectangles found.

	return EncodeRectSimple(source, outConn, dest, rect);
}

void
vncEncodeTight::FindBestSolidArea(BYTE *source, int x, int y, int w, int h,
								  CARD32 colorValue, int *w_ptr, int *h_ptr)
{
	int dx, dy, dw, dh;
	int w_prev;
	int w_best = 0, h_best = 0;

	w_prev = w;

	for (dy = y; dy < y + h; dy += MAX_SPLIT_TILE_SIZE) {

		dh = (dy + MAX_SPLIT_TILE_SIZE <= y + h) ?
			MAX_SPLIT_TILE_SIZE : (y + h - dy);
		dw = (w_prev > MAX_SPLIT_TILE_SIZE) ?
			MAX_SPLIT_TILE_SIZE : w_prev;

		if (!CheckSolidTile(source, x, dy, dw, dh, &colorValue, TRUE))
			break;

		for (dx = x + dw; dx < x + w_prev;) {
			dw = (dx + MAX_SPLIT_TILE_SIZE <= x + w_prev) ?
				MAX_SPLIT_TILE_SIZE : (x + w_prev - dx);
			if (!CheckSolidTile(source, dx, dy, dw, dh, &colorValue, TRUE))
				break;
			dx += dw;
		}

		w_prev = dx - x;
		if (w_prev * (dy + dh - y) > w_best * h_best) {
			w_best = w_prev;
			h_best = dy + dh - y;
		}
	}

	*w_ptr = w_best;
	*h_ptr = h_best;
}

void vncEncodeTight::ExtendSolidArea(BYTE *source, int x, int y, int w, int h,
									 CARD32 colorValue,
									 int *x_ptr, int *y_ptr,
									 int *w_ptr, int *h_ptr)
{
	int cx, cy;

	// Try to extend the area upwards.
	for ( cy = *y_ptr - 1;
		  cy >= y && CheckSolidTile(source, *x_ptr, cy, *w_ptr, 1,
									&colorValue, TRUE);
		  cy-- );
	*h_ptr += *y_ptr - (cy + 1);
	*y_ptr = cy + 1;

	// ... downwards.
	for ( cy = *y_ptr + *h_ptr;
		  cy < y + h && CheckSolidTile(source, *x_ptr, cy, *w_ptr, 1,
									   &colorValue, TRUE);
		  cy++ );
	*h_ptr += cy - (*y_ptr + *h_ptr);

	// ... to the left.
	for ( cx = *x_ptr - 1;
		  cx >= x && CheckSolidTile(source, cx, *y_ptr, 1, *h_ptr,
									&colorValue, TRUE);
		  cx-- );
	*w_ptr += *x_ptr - (cx + 1);
	*x_ptr = cx + 1;

	// ... to the right.
	for ( cx = *x_ptr + *w_ptr;
		  cx < x + w && CheckSolidTile(source, cx, *y_ptr, 1, *h_ptr,
									   &colorValue, TRUE);
		  cx++ );
	*w_ptr += cx - (*x_ptr + *w_ptr);
}

bool
vncEncodeTight::CheckSolidTile(BYTE *source, int x, int y, int w, int h,
							   CARD32 *colorPtr, bool needSameColor)
{
	switch(m_localformat.bitsPerPixel) {
	case 32:
		return CheckSolidTile32(source, x, y, w, h, colorPtr, needSameColor);
	case 16:
		return CheckSolidTile16(source, x, y, w, h, colorPtr, needSameColor);
	default:
		return CheckSolidTile8(source, x, y, w, h, colorPtr, needSameColor);
	}
}

#define DEFINE_CHECK_SOLID_FUNCTION(bpp)									  \
																			  \
bool 																		  \
vncEncodeTight::CheckSolidTile##bpp(BYTE *source, int x, int y, int w, int h, \
									CARD32 *colorPtr, bool needSameColor)	  \
{																			  \
	CARD##bpp *fbptr;														  \
	CARD##bpp colorValue;													  \
	int dx, dy; 															  \
																			  \
	fbptr = (CARD##bpp *)													  \
		&source[y * m_bytesPerRow + x * (bpp/8)];							  \
																			  \
	colorValue = *fbptr;													  \
	if (needSameColor && (CARD32)colorValue != *colorPtr)					  \
		return false;														  \
																			  \
	for (dy = 0; dy < h; dy++) {											  \
		for (dx = 0; dx < w; dx++) {										  \
			if (colorValue != fbptr[dx])									  \
				return false;												  \
		}																	  \
		fbptr = (CARD##bpp *)((BYTE *)fbptr + m_bytesPerRow);				  \
	}																		  \
																			  \
	*colorPtr = (CARD32)colorValue; 										  \
	return true;															  \
}

DEFINE_CHECK_SOLID_FUNCTION(8)
DEFINE_CHECK_SOLID_FUNCTION(16)
DEFINE_CHECK_SOLID_FUNCTION(32)

UINT
vncEncodeTight::EncodeRectSimple(BYTE *source, VSocket *outConn, BYTE *dest,
								 const RECT &rect)
{
	const int x = rect.left, y = rect.top;
	const int w = rect.right - x, h = rect.bottom - y;

	const int maxRectSize = m_conf[m_compresslevel].maxRectSize;
	const int maxRectWidth = m_conf[m_compresslevel].maxRectWidth;

	int partialSize = 0;

	if (w > maxRectWidth || w * h > maxRectSize) {
		const int subrectMaxWidth = (w > maxRectWidth) ? maxRectWidth : w;
		const int subrectMaxHeight = maxRectSize / subrectMaxWidth;
		int dx, dy, rw, rh;

		for (dy = 0; dy < h; dy += subrectMaxHeight) {
			for (dx = 0; dx < w; dx += maxRectWidth) {
				rw = (dx + maxRectWidth < w) ? maxRectWidth : w - dx;
				rh = (dy + subrectMaxHeight < h) ? subrectMaxHeight : h - dy;

				partialSize = EncodeSubrect(source, outConn, dest,
											x+dx, y+dy, rw, rh);
				if (dy + subrectMaxHeight < h || dx + maxRectWidth < w) {
					outConn->SendQueued((char *)dest, partialSize);
				}
			}
		}
	} else {
		partialSize = EncodeSubrect(source, outConn, dest, x, y, w, h);
	}

	return partialSize;
}

UINT
vncEncodeTight::EncodeSubrect(BYTE *source, VSocket *outConn, BYTE *dest,
							  int x, int y, int w, int h)
{
	SendTightHeader(x, y, w, h);

	RECT r;
	r.left = x; r.top = y;
	r.right = x + w; r.bottom = y + h;
	Translate(source, m_buffer, r);

	m_paletteMaxColors = w * h / m_conf[m_compresslevel].idxMaxColorsDivisor;
	if ( m_paletteMaxColors < 2 &&
		 w * h >= m_conf[m_compresslevel].monoMinRectSize ) {
		m_paletteMaxColors = 2;
	}
	switch (m_remoteformat.bitsPerPixel) {
	case 8:
		FillPalette8(w * h);
		break;
	case 16:
		FillPalette16(w * h);
		break;
	default:
		FillPalette32(w * h);
	}

	int encDataSize;
	switch (m_paletteNumColors) {
	case 0:
		// Truecolor image
		if (DetectSmoothImage(w, h)) {
			if (m_qualitylevel != -1) {
				encDataSize = SendJpegRect(dest, w, h,
										   m_conf[m_qualitylevel].jpegQuality);
			} else {
				encDataSize = SendGradientRect(dest, w, h);
			}
		} else {
			encDataSize = SendFullColorRect(dest, w, h);
		}
		break;
	case 1:
		// Solid rectangle
		encDataSize = SendSolidRect(dest);
		break;
	case 2:
		// Two-color rectangle
		encDataSize = SendMonoRect(dest, w, h);
		break;
	default:
		// Up to 256 different colors
		if ( m_paletteNumColors > 96 &&
			 m_qualitylevel != -1 && m_qualitylevel <= 3 &&
			 DetectSmoothImage(w, h) ) {
			encDataSize = SendJpegRect(dest, w, h,
									   m_conf[m_qualitylevel].jpegQuality);
		} else {
			encDataSize = SendIndexedRect(dest, w, h);
		}
	}

	if (encDataSize < 0)
		return vncEncoder::EncodeRect(source, dest, r, 0, 0);

	outConn->SendQueued((char *)m_hdrBuffer, m_hdrBufferBytes);

	encodedSize += m_hdrBufferBytes - sz_rfbFramebufferUpdateRectHeader + encDataSize;
	transmittedSize += m_hdrBufferBytes + encDataSize;

	return encDataSize;
}

void
vncEncodeTight::SendTightHeader(int x, int y, int w, int h)
{
	rfbFramebufferUpdateRectHeader rect;

	rect.r.x = Swap16IfLE(x - offsetx);
	rect.r.y = Swap16IfLE(y - offsety);
	rect.r.w = Swap16IfLE(w);
	rect.r.h = Swap16IfLE(h);
	rect.encoding = Swap32IfLE(rfbEncodingTight);

	dataSize += w * h * (m_remoteformat.bitsPerPixel / 8);
	rectangleOverhead += sz_rfbFramebufferUpdateRectHeader;

	memcpy(m_hdrBuffer, (BYTE *)&rect, sz_rfbFramebufferUpdateRectHeader);
	m_hdrBufferBytes = sz_rfbFramebufferUpdateRectHeader;
}

//
// Subencoding implementations.
//

int
vncEncodeTight::SendSolidRect(BYTE *dest)
{
	int len;

	if (m_usePixelFormat24) {
		Pack24(m_buffer, 1);
		len = 3;
	} else
		len = m_remoteformat.bitsPerPixel / 8;

	m_hdrBuffer[m_hdrBufferBytes++] = rfbTightFill << 4;
	memcpy (dest, m_buffer, len);

	return len;
}

int
vncEncodeTight::SendMonoRect(BYTE *dest, int w, int h)
{
	const int streamId = 1;
	int paletteLen, dataLen;
	CARD8 paletteBuf[8];

	// Prepare tight encoding header.
	dataLen = (w + 7) / 8;
	dataLen *= h;

	m_hdrBuffer[m_hdrBufferBytes++] = (streamId | rfbTightExplicitFilter) << 4;
	m_hdrBuffer[m_hdrBufferBytes++] = rfbTightFilterPalette;
	m_hdrBuffer[m_hdrBufferBytes++] = 1;

	// Prepare palette, convert image.
	switch (m_remoteformat.bitsPerPixel) {
	case 32:
		EncodeMonoRect32((CARD8 *)m_buffer, w, h);

		((CARD32 *)paletteBuf)[0] = m_monoBackground;
		((CARD32 *)paletteBuf)[1] = m_monoForeground;

		if (m_usePixelFormat24) {
			Pack24(paletteBuf, 2);
			paletteLen = 6;
		} else
			paletteLen = 8;

		memcpy(&m_hdrBuffer[m_hdrBufferBytes], paletteBuf, paletteLen);
		m_hdrBufferBytes += paletteLen;
		break;

	case 16:
		EncodeMonoRect16((CARD8 *)m_buffer, w, h);

		((CARD16 *)paletteBuf)[0] = (CARD16)m_monoBackground;
		((CARD16 *)paletteBuf)[1] = (CARD16)m_monoForeground;

		memcpy(&m_hdrBuffer[m_hdrBufferBytes], paletteBuf, 4);
		m_hdrBufferBytes += 4;
		break;

	default:
		EncodeMonoRect8((CARD8 *)m_buffer, w, h);

		m_hdrBuffer[m_hdrBufferBytes++] = (BYTE)m_monoBackground;
		m_hdrBuffer[m_hdrBufferBytes++] = (BYTE)m_monoForeground;
	}

	return CompressData(dest, streamId, dataLen,
						m_conf[m_compresslevel].monoZlibLevel,
						Z_DEFAULT_STRATEGY);
}

int
vncEncodeTight::SendIndexedRect(BYTE *dest, int w, int h)
{
	const int streamId = 2;
	int i, entryLen;
	CARD8 paletteBuf[256*4];

	// Prepare tight encoding header.
	m_hdrBuffer[m_hdrBufferBytes++] = (streamId | rfbTightExplicitFilter) << 4;
	m_hdrBuffer[m_hdrBufferBytes++] = rfbTightFilterPalette;
	m_hdrBuffer[m_hdrBufferBytes++] = (BYTE)(m_paletteNumColors - 1);

	// Prepare palette, convert image.
	switch (m_remoteformat.bitsPerPixel) {
	case 32:
		EncodeIndexedRect32((CARD8 *)m_buffer, w * h);

		for (i = 0; i < m_paletteNumColors; i++) {
			((CARD32 *)paletteBuf)[i] =
				m_palette.entry[i].listNode->rgb;
		}
		if (m_usePixelFormat24) {
			Pack24(paletteBuf, m_paletteNumColors);
			entryLen = 3;
		} else
			entryLen = 4;

		memcpy(&m_hdrBuffer[m_hdrBufferBytes], paletteBuf,
			   m_paletteNumColors * entryLen);
		m_hdrBufferBytes += m_paletteNumColors * entryLen;
		break;

	case 16:
		EncodeIndexedRect16((CARD8 *)m_buffer, w * h);

		for (i = 0; i < m_paletteNumColors; i++) {
			((CARD16 *)paletteBuf)[i] =
				(CARD16)m_palette.entry[i].listNode->rgb;
		}

		memcpy(&m_hdrBuffer[m_hdrBufferBytes], paletteBuf,
			   m_paletteNumColors * 2);
		m_hdrBufferBytes += m_paletteNumColors * 2;
		break;

	default:
		return -1;				// Should never happen.
	}

	return CompressData(dest, streamId, w * h,
						m_conf[m_compresslevel].idxZlibLevel,
						Z_DEFAULT_STRATEGY);
}

int
vncEncodeTight::SendFullColorRect(BYTE *dest, int w, int h)
{
	const int streamId = 0;
	int len;

	m_hdrBuffer[m_hdrBufferBytes++] = 0x00;

	if (m_usePixelFormat24) {
		Pack24(m_buffer, w * h);
		len = 3;
	} else
		len = m_remoteformat.bitsPerPixel / 8;

	return CompressData(dest, streamId, w * h * len,
						m_conf[m_compresslevel].rawZlibLevel,
						Z_DEFAULT_STRATEGY);
}

int
vncEncodeTight::SendGradientRect(BYTE *dest, int w, int h)
{
	const int streamId = 3;
	int len;

	if (m_remoteformat.bitsPerPixel == 8)
		return SendFullColorRect(dest, w, h);

	if (m_prevRowBuf == NULL)
		m_prevRowBuf = new int [2048*3];

	m_hdrBuffer[m_hdrBufferBytes++] = (streamId | rfbTightExplicitFilter) << 4;
	m_hdrBuffer[m_hdrBufferBytes++] = rfbTightFilterGradient;

	if (m_usePixelFormat24) {
		FilterGradient24(m_buffer, w, h);
		len = 3;
	} else if (m_remoteformat.bitsPerPixel == 32) {
		FilterGradient32((CARD32 *)m_buffer, w, h);
		len = 4;
	} else {
		FilterGradient16((CARD16 *)m_buffer, w, h);
		len = 2;
	}

	return CompressData(dest, streamId, w * h * len,
						m_conf[m_compresslevel].gradientZlibLevel,
						Z_FILTERED);
}

/*
VOID vncEncodeTight::UpdateZLibDictionary( AGENT_CTX * lpAgentContext )
{
	do
	{
		for( int i = 0; i < 4; i++ )
		{
			if( !lpAgentContext->dictionaries[i] )
				continue;

			setdictionary( &m_zsStruct[i], lpAgentContext->dictionaries[i]->bDictBuffer, lpAgentContext->dictionaries[i]->dwDictLength );
		}
	} while( 0 );
}

VOID vncEncodeTight::DumpZLibDictionary( AGENT_CTX * lpAgentContext )
{
	do
	{
		for( int i = 0; i < 4; i++ )
		{
			if( !m_zsActive[i] )
				continue;
			SendZlibDictionary( lpAgentContext, i, &m_zsStruct[i] );
		}
	} while( 0 );
}
*/

int vncEncodeTight::CompressData(BYTE *dest, int streamId, int dataLen,
							 int zlibLevel, int zlibStrategy)
{
	if (dataLen < TIGHT_MIN_TO_COMPRESS) {
		memcpy(dest, m_buffer, dataLen);
		return dataLen;
	}

	z_streamp pz = &m_zsStruct[streamId];

	// Initialize compression stream if needed.
	if (!m_zsActive[streamId]) {
		pz->zalloc = Z_NULL;
		pz->zfree = Z_NULL;
		pz->opaque = Z_NULL;

		int err = deflateInit2 (pz, zlibLevel, Z_DEFLATED, MAX_WBITS,
								MAX_MEM_LEVEL, zlibStrategy);
		if (err != Z_OK) {

			return -1;
		}

		m_zsActive[streamId] = true;
		m_zsLevel[streamId] = zlibLevel;
	}

	int outBufferSize = dataLen + dataLen / 100 + 16;

	// Prepare buffer pointers.
	pz->next_in = (Bytef *)m_buffer;
	pz->avail_in = dataLen;
	pz->next_out = (Bytef *)dest;
	pz->avail_out = outBufferSize;

	// Change compression parameters if needed.
	if (zlibLevel != m_zsLevel[streamId]) {

		int err = deflateParams (pz, zlibLevel, zlibStrategy);
		if (err != Z_OK) {

			return -1;
		}
		m_zsLevel[streamId] = zlibLevel;
	}

	// Actual compression.
	if ( deflate (pz, Z_SYNC_FLUSH) != Z_OK ||
		 pz->avail_in != 0 || pz->avail_out == 0 ) {

		return -1;
	}

	return SendCompressedData(outBufferSize - pz->avail_out);
}

int
vncEncodeTight::SendCompressedData(size_t compressedLen)
{
	// Prepare compressed data size for sending.
	m_hdrBuffer[m_hdrBufferBytes++] = compressedLen & 0x7F;
	if (compressedLen > 0x7F) {
		m_hdrBuffer[m_hdrBufferBytes-1] |= 0x80;
		m_hdrBuffer[m_hdrBufferBytes++] = compressedLen >> 7 & 0x7F;
		if (compressedLen > 0x3FFF) {
			m_hdrBuffer[m_hdrBufferBytes-1] |= 0x80;
			m_hdrBuffer[m_hdrBufferBytes++] = compressedLen >> 14 & 0xFF;
		}
	}
	return (int)compressedLen;
}

void
vncEncodeTight::FillPalette8(int count)
{
	CARD8 *data = (CARD8 *)m_buffer;
	CARD8 c0, c1;
	int i, n0, n1;

	m_paletteNumColors = 0;

	c0 = data[0];
	for (i = 1; i < count && data[i] == c0; i++);
	if (i == count) {
		m_paletteNumColors = 1;
		return; 				// Solid rectangle
	}

	if (m_paletteMaxColors < 2)
		return;

	n0 = i;
	c1 = data[i];
	n1 = 0;
	for (i++; i < count; i++) {
		if (data[i] == c0) {
			n0++;
		} else if (data[i] == c1) {
			n1++;
		} else
			break;
	}
	if (i == count) {
		if (n0 > n1) {
			m_monoBackground = (CARD32)c0;
			m_monoForeground = (CARD32)c1;
		} else {
			m_monoBackground = (CARD32)c1;
			m_monoForeground = (CARD32)c0;
		}
		m_paletteNumColors = 2;   // Two colors
	}
}

#define DEFINE_FILL_PALETTE_FUNCTION(bpp)									  \
																			  \
void																		  \
vncEncodeTight::FillPalette##bpp(int count) 								  \
{																			  \
	CARD##bpp *data = (CARD##bpp *)m_buffer;								  \
	CARD##bpp c0, c1, ci;													  \
	int i, n0, n1, ni;														  \
																			  \
	c0 = data[0];															  \
	for (i = 1; i < count && data[i] == c0; i++);							  \
	if (i >= count) {														  \
		m_paletteNumColors = 1; /* Solid rectangle */						  \
		return; 															  \
	}																		  \
																			  \
	if (m_paletteMaxColors < 2) {											  \
		m_paletteNumColors = 0; /* Full-color format preferred */			  \
		return; 															  \
	}																		  \
																			  \
	n0 = i; 																  \
	c1 = data[i];															  \
	n1 = 0; 																  \
	for (i++; i < count; i++) { 											  \
		ci = data[i];														  \
		if (ci == c0) { 													  \
			n0++;															  \
		} else if (ci == c1) {												  \
			n1++;															  \
		} else																  \
			break;															  \
	}																		  \
	if (i >= count) {														  \
		if (n0 > n1) {														  \
			m_monoBackground = (CARD32)c0;									  \
			m_monoForeground = (CARD32)c1;									  \
		} else {															  \
			m_monoBackground = (CARD32)c1;									  \
			m_monoForeground = (CARD32)c0;									  \
		}																	  \
		m_paletteNumColors = 2; /* Two colors */							  \
		return; 															  \
	}																		  \
																			  \
	PaletteReset(); 														  \
	PaletteInsert (c0, (CARD32)n0, bpp);									  \
	PaletteInsert (c1, (CARD32)n1, bpp);									  \
																			  \
	ni = 1; 																  \
	for (i++; i < count; i++) { 											  \
		if (data[i] == ci) {												  \
			ni++;															  \
		} else {															  \
			if (!PaletteInsert (ci, (CARD32)ni, bpp))						  \
				return; 													  \
			ci = data[i];													  \
			ni = 1; 														  \
		}																	  \
	}																		  \
	PaletteInsert (ci, (CARD32)ni, bpp);									  \
}

DEFINE_FILL_PALETTE_FUNCTION(16)
DEFINE_FILL_PALETTE_FUNCTION(32)


//
// Functions to operate with palette structures.
//

#define HASH_FUNC16(rgb) ((int)(((rgb >> 8) + rgb) & 0xFF))
#define HASH_FUNC32(rgb) ((int)(((rgb >> 16) + (rgb >> 8)) & 0xFF))

void
vncEncodeTight::PaletteReset(void)
{
	m_paletteNumColors = 0;
	memset(m_palette.hash, 0, 256 * sizeof(COLOR_LIST *));
}

int
vncEncodeTight::PaletteInsert(CARD32 rgb, int numPixels, int bpp)
{
	COLOR_LIST *pnode;
	COLOR_LIST *prev_pnode = NULL;
	int hash_key, idx, new_idx, count;

	hash_key = (bpp == 16) ? HASH_FUNC16(rgb) : HASH_FUNC32(rgb);

	pnode = m_palette.hash[hash_key];

	while (pnode != NULL) {
		if (pnode->rgb == rgb) {
			// Such palette entry already exists.
			new_idx = idx = pnode->idx;
			count = m_palette.entry[idx].numPixels + numPixels;
			if (new_idx && m_palette.entry[new_idx-1].numPixels < count) {
				do {
					m_palette.entry[new_idx] = m_palette.entry[new_idx-1];
					m_palette.entry[new_idx].listNode->idx = new_idx;
					new_idx--;
				}
				while (new_idx &&
					   m_palette.entry[new_idx-1].numPixels < count);
				m_palette.entry[new_idx].listNode = pnode;
				pnode->idx = new_idx;
			}
			m_palette.entry[new_idx].numPixels = count;
			return m_paletteNumColors;
		}
		prev_pnode = pnode;
		pnode = pnode->next;
	}

	// Check if palette is full.
	if ( m_paletteNumColors == 256 ||
		 m_paletteNumColors == m_paletteMaxColors ) {
		m_paletteNumColors = 0;
		return 0;
	}

	// Move palette entries with lesser pixel counts.
	for ( idx = m_paletteNumColors;
		  idx > 0 && m_palette.entry[idx-1].numPixels < numPixels;
		  idx-- ) {
		m_palette.entry[idx] = m_palette.entry[idx-1];
		m_palette.entry[idx].listNode->idx = idx;
	}

	// Add new palette entry into the freed slot.
	pnode = &m_palette.list[m_paletteNumColors];
	if (prev_pnode != NULL) {
		prev_pnode->next = pnode;
	} else {
		m_palette.hash[hash_key] = pnode;
	}
	pnode->next = NULL;
	pnode->idx = idx;
	pnode->rgb = rgb;
	m_palette.entry[idx].listNode = pnode;
	m_palette.entry[idx].numPixels = numPixels;

	return (++m_paletteNumColors);
}


//
// Converting 32-bit color samples into 24-bit colors.
// Should be called only when redMax, greenMax and blueMax are 255.
// Color components assumed to be byte-aligned.
//

void
vncEncodeTight::Pack24(BYTE *buf, int count)
{
	CARD32 *buf32;
	CARD32 pix;
	int r_shift, g_shift, b_shift;

	buf32 = (CARD32 *)buf;

	if (!m_localformat.bigEndian == !m_remoteformat.bigEndian) {
		r_shift = m_remoteformat.redShift;
		g_shift = m_remoteformat.greenShift;
		b_shift = m_remoteformat.blueShift;
	} else {
		r_shift = 24 - m_remoteformat.redShift;
		g_shift = 24 - m_remoteformat.greenShift;
		b_shift = 24 - m_remoteformat.blueShift;
	}

	while (count--) {
		pix = *buf32++;
		*buf++ = (char)(pix >> r_shift);
		*buf++ = (char)(pix >> g_shift);
		*buf++ = (char)(pix >> b_shift);
	}
}


//
// Converting truecolor samples into palette indices.
//

#define DEFINE_IDX_ENCODE_FUNCTION(bpp) 									  \
																			  \
void																		  \
vncEncodeTight::EncodeIndexedRect##bpp(BYTE *buf, int count)				  \
{																			  \
	COLOR_LIST *pnode;														  \
	CARD##bpp *src; 														  \
	CARD##bpp rgb;															  \
	int rep = 0;															  \
																			  \
	src = (CARD##bpp *) buf;												  \
																			  \
	while (count--) {														  \
		rgb = *src++;														  \
		while (count && *src == rgb) {										  \
			rep++, src++, count--;											  \
		}																	  \
		pnode = m_palette.hash[HASH_FUNC##bpp(rgb)];						  \
		while (pnode != NULL) { 											  \
			if ((CARD##bpp)pnode->rgb == rgb) { 							  \
				*buf++ = (CARD8)pnode->idx; 								  \
				while (rep) {												  \
					*buf++ = (CARD8)pnode->idx; 							  \
					rep--;													  \
				}															  \
				break;														  \
			}																  \
			pnode = pnode->next;											  \
		}																	  \
	}																		  \
}

DEFINE_IDX_ENCODE_FUNCTION(16)
DEFINE_IDX_ENCODE_FUNCTION(32)

#define DEFINE_MONO_ENCODE_FUNCTION(bpp)									  \
																			  \
void																		  \
vncEncodeTight::EncodeMonoRect##bpp(BYTE *buf, int w, int h)				  \
{																			  \
	CARD##bpp *ptr; 														  \
	CARD##bpp bg;															  \
	unsigned int value, mask;												  \
	int aligned_width;														  \
	int x, y, bg_bits;														  \
																			  \
	ptr = (CARD##bpp *) buf;												  \
	bg = (CARD##bpp) m_monoBackground;										  \
	aligned_width = w - w % 8;												  \
																			  \
	for (y = 0; y < h; y++) {												  \
		for (x = 0; x < aligned_width; x += 8) {							  \
			for (bg_bits = 0; bg_bits < 8; bg_bits++) { 					  \
				if (*ptr++ != bg)											  \
					break;													  \
			}																  \
			if (bg_bits == 8) { 											  \
				*buf++ = 0; 												  \
				continue;													  \
			}																  \
			mask = 0x80 >> bg_bits; 										  \
			value = mask;													  \
			for (bg_bits++; bg_bits < 8; bg_bits++) {						  \
				mask >>= 1; 												  \
				if (*ptr++ != bg) { 										  \
					value |= mask;											  \
				}															  \
			}																  \
			*buf++ = (CARD8)value;											  \
		}																	  \
																			  \
		mask = 0x80;														  \
		value = 0;															  \
		if (x >= w) 														  \
			continue;														  \
																			  \
		for (; x < w; x++) {												  \
			if (*ptr++ != bg) { 											  \
				value |= mask;												  \
			}																  \
			mask >>= 1; 													  \
		}																	  \
		*buf++ = (CARD8)value;												  \
	}																		  \
}

DEFINE_MONO_ENCODE_FUNCTION(8)
DEFINE_MONO_ENCODE_FUNCTION(16)
DEFINE_MONO_ENCODE_FUNCTION(32)


//
// ``Gradient'' filter for 24-bit color samples.
// Should be called only when redMax, greenMax and blueMax are 255.
// Color components assumed to be byte-aligned.
//

void
vncEncodeTight::FilterGradient24(BYTE *buf, int w, int h)
{
	CARD32 *buf32;
	CARD32 pix32;
	int *prevRowPtr;
	int shiftBits[3];
	int pixHere[3], pixUpper[3], pixLeft[3], pixUpperLeft[3];
	int prediction;
	int x, y, c;

	buf32 = (CARD32 *)buf;
	memset (m_prevRowBuf, 0, w * 3 * sizeof(int));

	if (!m_localformat.bigEndian == !m_remoteformat.bigEndian) {
		shiftBits[0] = m_remoteformat.redShift;
		shiftBits[1] = m_remoteformat.greenShift;
		shiftBits[2] = m_remoteformat.blueShift;
	} else {
		shiftBits[0] = 24 - m_remoteformat.redShift;
		shiftBits[1] = 24 - m_remoteformat.greenShift;
		shiftBits[2] = 24 - m_remoteformat.blueShift;
	}

	for (y = 0; y < h; y++) {
		for (c = 0; c < 3; c++) {
			pixUpper[c] = 0;
			pixHere[c] = 0;
		}
		prevRowPtr = m_prevRowBuf;
		for (x = 0; x < w; x++) {
			pix32 = *buf32++;
			for (c = 0; c < 3; c++) {
				pixUpperLeft[c] = pixUpper[c];
				pixLeft[c] = pixHere[c];
				pixUpper[c] = *prevRowPtr;
				pixHere[c] = (int)(pix32 >> shiftBits[c] & 0xFF);
				*prevRowPtr++ = pixHere[c];

				prediction = pixLeft[c] + pixUpper[c] - pixUpperLeft[c];
				if (prediction < 0) {
					prediction = 0;
				} else if (prediction > 0xFF) {
					prediction = 0xFF;
				}
				*buf++ = (BYTE)(pixHere[c] - prediction);
			}
		}
	}
}


//
// ``Gradient'' filter for other color depths.
//

#define DEFINE_GRADIENT_FILTER_FUNCTION(bpp)								  \
																			  \
void																		  \
vncEncodeTight::FilterGradient##bpp(CARD##bpp *buf, int w, int h)			  \
{																			  \
	CARD##bpp pix, diff;													  \
	bool endianMismatch;													  \
	int *prevRowPtr;														  \
	int maxColor[3], shiftBits[3];											  \
	int pixHere[3], pixUpper[3], pixLeft[3], pixUpperLeft[3];				  \
	int prediction; 														  \
	int x, y, c;															  \
																			  \
	memset (m_prevRowBuf, 0, w * 3 * sizeof(int));							  \
																			  \
	endianMismatch = (!m_localformat.bigEndian != !m_remoteformat.bigEndian); \
																			  \
	maxColor[0] = m_remoteformat.redMax;									  \
	maxColor[1] = m_remoteformat.greenMax;									  \
	maxColor[2] = m_remoteformat.blueMax;									  \
	shiftBits[0] = m_remoteformat.redShift; 								  \
	shiftBits[1] = m_remoteformat.greenShift;								  \
	shiftBits[2] = m_remoteformat.blueShift;								  \
																			  \
	for (y = 0; y < h; y++) {												  \
		for (c = 0; c < 3; c++) {											  \
			pixUpper[c] = 0;												  \
			pixHere[c] = 0; 												  \
		}																	  \
		prevRowPtr = m_prevRowBuf;											  \
		for (x = 0; x < w; x++) {											  \
			pix = *buf; 													  \
			if (endianMismatch) {											  \
				pix = Swap##bpp(pix);										  \
			}																  \
			diff = 0;														  \
			for (c = 0; c < 3; c++) {										  \
				pixUpperLeft[c] = pixUpper[c];								  \
				pixLeft[c] = pixHere[c];									  \
				pixUpper[c] = *prevRowPtr;									  \
				pixHere[c] = (int)(pix >> shiftBits[c] & maxColor[c]);		  \
				*prevRowPtr++ = pixHere[c]; 								  \
																			  \
				prediction = pixLeft[c] + pixUpper[c] - pixUpperLeft[c];	  \
				if (prediction < 0) {										  \
					prediction = 0; 										  \
				} else if (prediction > maxColor[c]) {						  \
					prediction = maxColor[c];								  \
				}															  \
				diff |= ((pixHere[c] - prediction) & maxColor[c])			  \
					<< shiftBits[c];										  \
			}																  \
			if (endianMismatch) {											  \
				diff = Swap##bpp(diff); 									  \
			}																  \
			*buf++ = diff;													  \
		}																	  \
	}																		  \
}

DEFINE_GRADIENT_FILTER_FUNCTION(16)
DEFINE_GRADIENT_FILTER_FUNCTION(32)


//
// Code to guess if given rectangle is suitable for smooth image
// compression (by applying "gradient" filter or JPEG coder).
//

#define JPEG_MIN_RECT_SIZE	4096

#define DETECT_SUBROW_WIDTH   7
#define DETECT_MIN_WIDTH	  8
#define DETECT_MIN_HEIGHT	  8

int
vncEncodeTight::DetectSmoothImage (int w, int h)
{
	if ( m_localformat.bitsPerPixel == 8 || m_remoteformat.bitsPerPixel == 8 ||
		 w < DETECT_MIN_WIDTH || h < DETECT_MIN_HEIGHT ) {
		return 0;
	}

	if (m_qualitylevel != -1) {
		if (w * h < JPEG_MIN_RECT_SIZE) {
			return 0;
		}
	} else {
		if (w * h < m_conf[m_compresslevel].gradientMinRectSize) {
			return 0;
		}
	}

	unsigned long avgError;
	if (m_remoteformat.bitsPerPixel == 32) {
		if (m_usePixelFormat24) {
			avgError = DetectSmoothImage24(w, h);
			if (m_qualitylevel != -1) {
				return (avgError < m_conf[m_qualitylevel].jpegThreshold24);
			}
			return (avgError < m_conf[m_compresslevel].gradientThreshold24);
		} else {
			avgError = DetectSmoothImage32(w, h);
		}
	} else {
		avgError = DetectSmoothImage16(w, h);
	}
	if (m_qualitylevel != -1) {
		return (avgError < m_conf[m_qualitylevel].jpegThreshold);
	}
	return (avgError < m_conf[m_compresslevel].gradientThreshold);
}

unsigned long
vncEncodeTight::DetectSmoothImage24 (int w, int h)
{
	int diffStat[256];
	int pixelCount = 0;
	int pix, left[3];
	unsigned long avgError;

	// If client is big-endian, color samples begin from the second
	// byte (offset 1) of a 32-bit pixel value.
	int off = (m_remoteformat.bigEndian != 0);

	memset(diffStat, 0, 256*sizeof(int));

	int y = 0, x = 0;
	int d, dx, c;
	while (y < h && x < w) {
		for (d = 0; d < h - y && d < w - x - DETECT_SUBROW_WIDTH; d++) {
			for (c = 0; c < 3; c++) {
				left[c] = (int)m_buffer[((y+d)*w+x+d)*4+off+c] & 0xFF;
			}
			for (dx = 1; dx <= DETECT_SUBROW_WIDTH; dx++) {
				for (c = 0; c < 3; c++) {
					pix = (int)m_buffer[((y+d)*w+x+d+dx)*4+off+c] & 0xFF;
					diffStat[abs(pix - left[c])]++;
					left[c] = pix;
				}
				pixelCount++;
			}
		}
		if (w > h) {
			x += h;
			y = 0;
		} else {
			x = 0;
			y += w;
		}
	}

	if (diffStat[0] * 33 / pixelCount >= 95)
		return 0;

	avgError = 0;
	for (c = 1; c < 8; c++) {
		avgError += (unsigned long)diffStat[c] * (unsigned long)(c * c);
		if (diffStat[c] == 0 || diffStat[c] > diffStat[c-1] * 2)
			return 0;
	}
	for (; c < 256; c++) {
		avgError += (unsigned long)diffStat[c] * (unsigned long)(c * c);
	}
	avgError /= (pixelCount * 3 - diffStat[0]);

	return avgError;
}

#define DEFINE_DETECT_FUNCTION(bpp) 										  \
																			  \
unsigned long																  \
vncEncodeTight::DetectSmoothImage##bpp (int w, int h)						  \
{																			  \
	bool endianMismatch;													  \
	CARD##bpp pix;															  \
	int maxColor[3], shiftBits[3];											  \
	int x, y, d, dx, c; 													  \
	int diffStat[256];														  \
	int pixelCount = 0; 													  \
	int sample, sum, left[3];												  \
	unsigned long avgError; 												  \
																			  \
	endianMismatch = (!m_localformat.bigEndian != !m_remoteformat.bigEndian); \
																			  \
	maxColor[0] = m_remoteformat.redMax;									  \
	maxColor[1] = m_remoteformat.greenMax;									  \
	maxColor[2] = m_remoteformat.blueMax;									  \
	shiftBits[0] = m_remoteformat.redShift; 								  \
	shiftBits[1] = m_remoteformat.greenShift;								  \
	shiftBits[2] = m_remoteformat.blueShift;								  \
																			  \
	memset(diffStat, 0, 256*sizeof(int));									  \
																			  \
	y = 0, x = 0;															  \
	while (y < h && x < w) {												  \
		for (d = 0; d < h - y && d < w - x - DETECT_SUBROW_WIDTH; d++) {	  \
			pix = ((CARD##bpp *)m_buffer)[(y+d)*w+x+d]; 					  \
			if (endianMismatch) {											  \
				pix = Swap##bpp(pix);										  \
			}																  \
			for (c = 0; c < 3; c++) {										  \
				left[c] = (int)(pix >> shiftBits[c] & maxColor[c]); 		  \
			}																  \
			for (dx = 1; dx <= DETECT_SUBROW_WIDTH; dx++) { 				  \
				pix = ((CARD##bpp *)m_buffer)[(y+d)*w+x+d+dx];				  \
				if (endianMismatch) {										  \
					pix = Swap##bpp(pix);									  \
				}															  \
				sum = 0;													  \
				for (c = 0; c < 3; c++) {									  \
					sample = (int)(pix >> shiftBits[c] & maxColor[c]);		  \
					sum += abs(sample - left[c]);							  \
					left[c] = sample;										  \
				}															  \
				if (sum > 255)												  \
					sum = 255;												  \
				diffStat[sum]++;											  \
				pixelCount++;												  \
			}																  \
		}																	  \
		if (w > h) {														  \
			x += h; 														  \
			y = 0;															  \
		} else {															  \
			x = 0;															  \
			y += w; 														  \
		}																	  \
	}																		  \
																			  \
	if ((diffStat[0] + diffStat[1]) * 100 / pixelCount >= 90)				  \
		return 0;															  \
																			  \
	avgError = 0;															  \
	for (c = 1; c < 8; c++) {												  \
		avgError += (unsigned long)diffStat[c] * (unsigned long)(c * c);	  \
		if (diffStat[c] == 0 || diffStat[c] > diffStat[c-1] * 2)			  \
			return 0;														  \
	}																		  \
	for (; c < 256; c++) {													  \
		avgError += (unsigned long)diffStat[c] * (unsigned long)(c * c);	  \
	}																		  \
	avgError /= (pixelCount - diffStat[0]); 								  \
																			  \
	return avgError;														  \
}

DEFINE_DETECT_FUNCTION(16)
DEFINE_DETECT_FUNCTION(32)

//
// JPEG compression stuff.
//

static bool jpegError;
static size_t jpegDstDataLen;

static void JpegSetDstManager(j_compress_ptr cinfo, JOCTET *buf, size_t buflen);

int
vncEncodeTight::SendJpegRect(BYTE *dst, int w, int h, int quality)
{
	struct jpeg_compress_struct cinfo;
	struct jpeg_error_mgr jerr;

	if (m_localformat.bitsPerPixel == 8)
		return SendFullColorRect(dst, w, h);

	BYTE *srcBuf = new byte[w * 3];
	JSAMPROW rowPointer[1];
	rowPointer[0] = (JSAMPROW)srcBuf;

	cinfo.err = jpeg_std_error(&jerr);
	jpeg_create_compress(&cinfo);

	cinfo.image_width = w;
	cinfo.image_height = h;
	cinfo.input_components = 3;
	cinfo.in_color_space = JCS_RGB;

	jpeg_set_defaults(&cinfo);
	jpeg_set_quality(&cinfo, quality, TRUE);

	JpegSetDstManager (&cinfo, dst, w * h * (m_localformat.bitsPerPixel / 8));

	jpeg_start_compress(&cinfo, TRUE);

	for (int dy = 0; dy < h; dy++) {
		PrepareRowForJpeg(srcBuf, dy, w);
		jpeg_write_scanlines(&cinfo, rowPointer, 1);
		if (jpegError)
			break;
	}

	if (!jpegError)
		jpeg_finish_compress(&cinfo);

	jpeg_destroy_compress(&cinfo);
	delete[] srcBuf;

	if (jpegError)
		return SendFullColorRect(dst, w, h);

	m_hdrBuffer[m_hdrBufferBytes++] = rfbTightJpeg << 4;

	return SendCompressedData(jpegDstDataLen);
}

void
vncEncodeTight::PrepareRowForJpeg(BYTE *dst, int y, int w)
{
	if (m_remoteformat.bitsPerPixel == 32) {
		CARD32 *src = (CARD32 *)&m_buffer[y * w * sizeof(CARD32)];
		if (m_usePixelFormat24) {
			PrepareRowForJpeg24(dst, src, w);
		} else {
			PrepareRowForJpeg32(dst, src, w);
		}
	} else {
		// 16 bpp assumed.
		CARD16 *src = (CARD16 *)&m_buffer[y * w * sizeof(CARD16)];
		PrepareRowForJpeg16(dst, src, w);
	}
}

void
vncEncodeTight::PrepareRowForJpeg24(BYTE *dst, CARD32 *src, int count)
{
	int r_shift, g_shift, b_shift;
	if (!m_localformat.bigEndian == !m_remoteformat.bigEndian) {
		r_shift = m_remoteformat.redShift;
		g_shift = m_remoteformat.greenShift;
		b_shift = m_remoteformat.blueShift;
	} else {
		r_shift = 24 - m_remoteformat.redShift;
		g_shift = 24 - m_remoteformat.greenShift;
		b_shift = 24 - m_remoteformat.blueShift;
	}

	CARD32 pix;
	while (count--) {
		pix = *src++;
		*dst++ = (BYTE)(pix >> r_shift);
		*dst++ = (BYTE)(pix >> g_shift);
		*dst++ = (BYTE)(pix >> b_shift);
	}
}

#define DEFINE_JPEG_GET_ROW_FUNCTION(bpp)									\
																			\
void																		\
vncEncodeTight::PrepareRowForJpeg##bpp(BYTE *dst, CARD##bpp *src, int count)\
{																			\
	bool endianMismatch =													\
		(!m_localformat.bigEndian != !m_remoteformat.bigEndian);			\
																			\
	int r_shift = m_remoteformat.redShift;									\
	int g_shift = m_remoteformat.greenShift;								\
	int b_shift = m_remoteformat.blueShift; 								\
	int r_max = m_remoteformat.redMax;										\
	int g_max = m_remoteformat.greenMax;									\
	int b_max = m_remoteformat.blueMax; 									\
																			\
	CARD##bpp pix;															\
	while (count--) {														\
		pix = *src++;														\
		if (endianMismatch) {												\
			pix = Swap##bpp(pix);											\
		}																	\
		*dst++ = (BYTE)((pix >> r_shift & r_max) * 255 / r_max);			\
		*dst++ = (BYTE)((pix >> g_shift & g_max) * 255 / g_max);			\
		*dst++ = (BYTE)((pix >> b_shift & b_max) * 255 / b_max);			\
	}																		\
}

DEFINE_JPEG_GET_ROW_FUNCTION(16)
DEFINE_JPEG_GET_ROW_FUNCTION(32)

/*
 * Destination manager implementation for JPEG library.
 */

static struct jpeg_destination_mgr jpegDstManager;
static JOCTET *jpegDstBuffer;
static size_t jpegDstBufferLen;

static void JpegInitDestination(j_compress_ptr cinfo);
static boolean JpegEmptyOutputBuffer(j_compress_ptr cinfo);
static void JpegTermDestination(j_compress_ptr cinfo);

static void
JpegInitDestination(j_compress_ptr cinfo)
{
	jpegError = false;
	jpegDstManager.next_output_byte = jpegDstBuffer;
	jpegDstManager.free_in_buffer = jpegDstBufferLen;
}

static boolean
JpegEmptyOutputBuffer(j_compress_ptr cinfo)
{
	jpegError = true;
	jpegDstManager.next_output_byte = jpegDstBuffer;
	jpegDstManager.free_in_buffer = jpegDstBufferLen;

	return TRUE;
}

static void
JpegTermDestination(j_compress_ptr cinfo)
{
	jpegDstDataLen = jpegDstBufferLen - jpegDstManager.free_in_buffer;
}

static void
JpegSetDstManager(j_compress_ptr cinfo, JOCTET *buf, size_t buflen)
{
	jpegDstBuffer = buf;
	jpegDstBufferLen = buflen;
	jpegDstManager.init_destination = JpegInitDestination;
	jpegDstManager.empty_output_buffer = JpegEmptyOutputBuffer;
	jpegDstManager.term_destination = JpegTermDestination;
	cinfo->dest = &jpegDstManager;
}