From e8b25fed588d0ea98d3c7480d5b16a2872e878ae Mon Sep 17 00:00:00 2001 From: Thorsten Engel Date: Wed, 17 May 2006 09:26:14 +0000 Subject: [PATCH] - reduce page faults by reusing zlib buffers during backup (on windows > 10.000/sec, now <200/sec) - reduce utf8file->ucs2file conversion load during backup to 1/3 git-svn-id: https://bacula.svn.sourceforge.net/svnroot/bacula/trunk@3024 91ce42f0-d328-0410-95d8-f526ca767f89 --- bacula/src/filed/backup.c | 41 +++++++++++++++++++++++++---- bacula/src/lib/winapi.c | 1 + bacula/src/win32/compat/compat.cpp | 42 ++++++++++++++++++++++++++++++ bacula/src/win32/compat/compat.h | 1 + 4 files changed, 80 insertions(+), 5 deletions(-) diff --git a/bacula/src/filed/backup.c b/bacula/src/filed/backup.c index 494b24add6..6088f157ad 100644 --- a/bacula/src/filed/backup.c +++ b/bacula/src/filed/backup.c @@ -643,6 +643,21 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *sign if (S_ISBLK(ff_pkt->statp.st_mode)) rsize = (rsize/512) * 512; #endif + +#ifdef HAVE_LIBZ + /* + * instead of using compress2 for every block (with 256KB alloc + free per block) + * we init the zlib once per file which leads to less pagefaults on large files (>64K) + */ + + z_stream zstream; + zstream.zalloc = Z_NULL; + zstream.zfree = Z_NULL; + zstream.opaque = Z_NULL; + zstream.state = Z_NULL; + + BOOL blibzInited = deflateInit(&zstream, ff_pkt->GZIP_level) == Z_OK; +#endif /* * Read the file data @@ -682,19 +697,25 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *sign #ifdef HAVE_LIBZ /* Do compression if turned on */ - if (!sparseBlock && ff_pkt->flags & FO_GZIP) { + if (!sparseBlock && (ff_pkt->flags & FO_GZIP) && blibzInited) { int zstat; compress_len = max_compress_len; Dmsg4(400, "cbuf=0x%x len=%u rbuf=0x%x len=%u\n", cbuf, compress_len, rbuf, sd->msglen); - /* NOTE! This call modifies compress_len !!! */ - if ((zstat=compress2((Bytef *)cbuf, &compress_len, - (const Bytef *)rbuf, (uLong)sd->msglen, - ff_pkt->GZIP_level)) != Z_OK) { + + zstream.next_in = (Bytef *)rbuf; + zstream.avail_in = sd->msglen; + zstream.next_out = (Bytef *)cbuf; + zstream.avail_out = compress_len; + + if ((zstat=deflate(&zstream, Z_FINISH)) != Z_STREAM_END) { Jmsg(jcr, M_FATAL, 0, _("Compression error: %d\n"), zstat); set_jcr_job_status(jcr, JS_ErrorTerminated); goto err; } + compress_len = zstream.total_out; + blibzInited = deflateReset (&zstream) == Z_OK; + Dmsg2(400, "compressed len=%d uncompressed len=%d\n", compress_len, sd->msglen); @@ -786,13 +807,23 @@ int send_data(JCR *jcr, int stream, FF_PKT *ff_pkt, DIGEST *digest, DIGEST *sign if (cipher_ctx) { crypto_cipher_free(cipher_ctx); } +#ifdef HAVE_LIBZ + /* Free the zlib stream */ + deflateEnd(&zstream); +#endif return 1; err: + /* Free the cipher context */ if (cipher_ctx) { crypto_cipher_free(cipher_ctx); } +#ifdef HAVE_LIBZ + /* Free the zlib stream */ + deflateEnd(&zstream); +#endif + sd->msg = msgsave; /* restore bnet buffer */ sd->msglen = 0; return 0; diff --git a/bacula/src/lib/winapi.c b/bacula/src/lib/winapi.c index 80fb5f7963..fe6b7eb007 100644 --- a/bacula/src/lib/winapi.c +++ b/bacula/src/lib/winapi.c @@ -248,6 +248,7 @@ InitWinAPIWrapper() break; } #endif /* WIN32_VSS */ + atexit (Win32ConvCleanupCache); } #else diff --git a/bacula/src/win32/compat/compat.cpp b/bacula/src/win32/compat/compat.cpp index 03d8835b2d..6c871331db 100644 --- a/bacula/src/win32/compat/compat.cpp +++ b/bacula/src/win32/compat/compat.cpp @@ -40,6 +40,29 @@ #define b_errno_win32 (1<<29) +/* UTF-8 to UCS2 path conversion is expensive, + so we cache the conversion. During backup the + conversion is called 3 times (lstat, attribs, open), + by using the cache this is reduced to 1 time */ + +POOLMEM* g_pWin32ConvUTF8Cache = get_pool_memory (PM_FNAME); +POOLMEM* g_pWin32ConvUCS2Cache = get_pool_memory (PM_FNAME); +static pthread_mutex_t Win32Convmutex = PTHREAD_MUTEX_INITIALIZER; + +void Win32ConvCleanupCache() +{ + if (g_pWin32ConvUTF8Cache) { + free_pool_memory (g_pWin32ConvUTF8Cache); + g_pWin32ConvUTF8Cache = NULL; + } + + if (g_pWin32ConvUCS2Cache) { + free_pool_memory (g_pWin32ConvUCS2Cache); + g_pWin32ConvUCS2Cache = NULL; + } +} + + /* to allow the usage of the original version in this file here */ #undef fputs @@ -349,6 +372,16 @@ wchar_win32_path(const char *name, wchar_t *win32_name) int make_win32_path_UTF8_2_wchar(POOLMEM **pszUCS, const char *pszUTF, BOOL* pBIsRawPath /*= NULL*/) { + P(Win32Convmutex); + /* if we find the utf8 string in cache, we use the cached ucs2 version */ + if (bstrcmp (pszUTF, g_pWin32ConvUTF8Cache)) { + int32_t nBufSize = sizeof_pool_memory(g_pWin32ConvUCS2Cache); + *pszUCS = check_pool_memory_size(*pszUCS, nBufSize); + wcscpy ((LPWSTR) *pszUCS, (LPWSTR) g_pWin32ConvUCS2Cache); + V(Win32Convmutex); + return nBufSize / sizeof (WCHAR); + } + /* helper to convert from utf-8 to UCS-2 and to complete a path for 32K path syntax */ int nRet = UTF8_2_wchar(pszUCS, pszUTF); @@ -360,6 +393,15 @@ make_win32_path_UTF8_2_wchar(POOLMEM **pszUCS, const char *pszUTF, BOOL* pBIsRaw *pBIsRawPath = FALSE; #endif + /* populate cache */ + int32_t nBufSize = sizeof_pool_memory(*pszUCS); + g_pWin32ConvUCS2Cache = check_pool_memory_size(g_pWin32ConvUCS2Cache, nBufSize); + wcscpy ((LPWSTR) g_pWin32ConvUCS2Cache, (LPWSTR) *pszUCS); + nBufSize = strlen (pszUTF)+1; + g_pWin32ConvUTF8Cache = check_pool_memory_size(g_pWin32ConvUTF8Cache, nBufSize); + bstrncpy (g_pWin32ConvUTF8Cache, pszUTF, nBufSize); + V(Win32Convmutex); + return nRet; } diff --git a/bacula/src/win32/compat/compat.h b/bacula/src/win32/compat/compat.h index 610e2d0298..81b58943ac 100644 --- a/bacula/src/win32/compat/compat.h +++ b/bacula/src/win32/compat/compat.h @@ -380,6 +380,7 @@ int win32_unlink(const char *filename); char* win32_cgets (char* buffer, int len); int WSA_Init(void); +void Win32ConvCleanupCache(); #ifdef HAVE_MINGW void closelog(); -- 2.39.5