Changes - j0ke.net Open Build Service

@@ -1,4 +1,78 @@ +Changes with nginx 1.7.11 24 Mar 2015 + + *) Change: the "sendfile" parameter of the "aio" directive is + deprecated; now nginx automatically uses AIO to pre-load data for + sendfile if both "aio" and "sendfile" directives are used. + + *) Feature: experimental thread pools support. + + *) Feature: the "proxy_request_buffering", "fastcgi_request_buffering", + "scgi_request_buffering", and "uwsgi_request_buffering" directives. + + *) Feature: request body filters experimental API. + + *) Feature: client SSL certificates support in mail proxy. + Thanks to Sven Peter, Franck Levionnois, and Filipe Da Silva. + + *) Feature: startup speedup when using the "hash ... consistent" + directive in the upstream block. + Thanks to Wai Keen Woon. + + *) Feature: debug logging into a cyclic memory buffer. + + *) Bugfix: in hash table handling. + Thanks to Chris West. + + *) Bugfix: in the "proxy_cache_revalidate" directive. + + *) Bugfix: SSL connections might hang if deferred accept or the + "proxy_protocol" parameter of the "listen" directive were used. + Thanks to James Hamlin. + + *) Bugfix: the $upstream_response_time variable might contain a wrong + value if the "image_filter" directive was used. + + *) Bugfix: in integer overflow handling. + Thanks to Régis Leroy. + + *) Bugfix: it was not possible to enable SSLv3 with LibreSSL. + + *) Bugfix: the "ignoring stale global SSL error ... called a function + you should not call" alerts appeared in logs when using LibreSSL. + + *) Bugfix: certificates specified by the "ssl_client_certificate" and + "ssl_trusted_certificate" directives were inadvertently used to + automatically construct certificate chains. + + +Changes with nginx 1.7.10 10 Feb 2015 + + *) Feature: the "use_temp_path" parameter of the "proxy_cache_path", + "fastcgi_cache_path", "scgi_cache_path", and "uwsgi_cache_path" + directives. + + *) Feature: the $upstream_header_time variable. + + *) Workaround: now on disk overflow nginx tries to write error logs once + a second only. + + *) Bugfix: the "try_files" directive did not ignore normal files while + testing directories. + Thanks to Damien Tournoud. + + *) Bugfix: alerts "sendfile() failed" if the "sendfile" directive was + used on OS X; the bug had appeared in 1.7.8. + + *) Bugfix: alerts "sem_post() failed" might appear in logs. + + *) Bugfix: nginx could not be built with musl libc. + Thanks to James Taylor. + + *) Bugfix: nginx could not be built on Tru64 UNIX. + Thanks to Goetz T. Fischer. + + Changes with nginx 1.7.9 23 Dec 2014 *) Feature: variables support in the "proxy_cache", "fastcgi_cache",

@@ -1,4 +1,81 @@ +Изменения в nginx 1.7.11 24.03.2015 + + *) Изменение: параметр sendfile директивы aio более не нужен; теперь + nginx автоматически использует AIO для подгрузки данных для sendfile, + если одновременно используются директивы aio и sendfile. + + *) Добавление: экспериментальная поддержка потоков. + + *) Добавление: директивы proxy_request_buffering, + fastcgi_request_buffering, scgi_request_buffering и + uwsgi_request_buffering. + + *) Добавление: экспериментальное API для обработки тела запроса. + + *) Добавление: проверка клиентских SSL-сертификатов в почтовом + прокси-сервере. + Спасибо Sven Peter, Franck Levionnois и Filipe Da Silva. + + *) Добавление: уменьшение времени запуска при использовании дирекивы + "hash ... consistent" в блоке upstream. + Спасибо Wai Keen Woon. + + *) Добавление: отладочное логгирование в кольцевой буфер в памяти. + + *) Исправление: в обработке хэш-таблиц. + Спасибо Chris West. + + *) Исправление: в директиве proxy_cache_revalidate. + + *) Исправление: SSL-соединения могли зависать, если использовался + отложенный accept или параметр proxy_protocol директивы listen. + Спасибо James Hamlin. + + *) Исправление: переменная $upstream_response_time могла содержать + неверное значение при использовании директивы image_filter. + + *) Исправление: в обработке целочисленных переполнений. + Спасибо Régis Leroy. + + *) Исправление: при использовании LibreSSL было невозможно включить + поддержку SSLv3. + + *) Исправление: при использовании LibreSSL в логах появлялись сообщения + "ignoring stale global SSL error ... called a function you should not + call". + + *) Исправление: сертификаты, указанные в директивах + ssl_client_certificate и ssl_trusted_certificate, использовались для + автоматического построения цепочек сертификатов. + + +Изменения в nginx 1.7.10 10.02.2015 + + *) Добавление: параметр use_temp_path директив proxy_cache_path, + fastcgi_cache_path, scgi_cache_path и uwsgi_cache_path. + + *) Добавление: переменная $upstream_header_time. + + *) Изменение: теперь при переполнении диска nginx пытается писать + error_log'и только раз в секунду. + + *) Исправление: директива try_files при тестировании каталогов не + игнорировала обычные файлы. + Спасибо Damien Tournoud. + + *) Исправление: при использовании директивы sendfile на OS X возникали + ошибки "sendfile() failed"; ошибка появилась в nginx 1.7.8. + + *) Исправление: в лог могли писаться сообщения "sem_post() failed". + + *) Исправление: nginx не собирался с musl libc. + Спасибо James Taylor. + + *) Исправление: nginx не собирался на Tru64 UNIX. + Спасибо Goetz T. Fischer. + + Изменения в nginx 1.7.9 23.12.2014 *) Добавление: директивы proxy_cache, fastcgi_cache, scgi_cache и

@@ -1,6 +1,6 @@ /* - * Copyright (C) 2002-2014 Igor Sysoev - * Copyright (C) 2011-2014 Nginx, Inc. + * Copyright (C) 2002-2015 Igor Sysoev + * Copyright (C) 2011-2015 Nginx, Inc. * All rights reserved. * * Redistribution and use in source and binary forms, with or without

@@ -41,11 +41,6 @@ ;; *) - case $USE_THREADS in - NO) OPENSSL_OPT="$OPENSSL_OPT no-threads" ;; - *) OPENSSL_OPT="$OPENSSL_OPT threads" ;; - esac - case $OPENSSL in /*) ngx_prefix="$OPENSSL/.openssl" ;; *) ngx_prefix="$PWD/$OPENSSL/.openssl" ;;

@@ -432,6 +432,12 @@ modules="$CORE_MODULES $EVENT_MODULES" +# thread pool module should be initialized after events +if [ $USE_THREADS = YES ]; then + modules="$modules $THREAD_POOL_MODULE" +fi + + if [ $USE_OPENSSL = YES ]; then modules="$modules $OPENSSL_MODULE" CORE_DEPS="$CORE_DEPS $OPENSSL_DEPS"

@@ -190,8 +190,7 @@ --without-poll_module) EVENT_POLL=NONE ;; --with-aio_module) EVENT_AIO=YES ;; - #--with-threads=*) USE_THREADS="$value" ;; - #--with-threads) USE_THREADS="pthreads" ;; + --with-threads) USE_THREADS=YES ;; --with-file-aio) NGX_FILE_AIO=YES ;; --with-ipv6) NGX_IPV6=YES ;; @@ -354,6 +353,8 @@ --with-poll_module enable poll module --without-poll_module disable poll module + --with-threads enable thread pool support + --with-file-aio enable file AIO support --with-ipv6 enable IPv6 support

@@ -100,7 +100,6 @@ . auto/feature if [ $ngx_found = yes ]; then - have=NGX_HAVE_SENDFILE . auto/have CORE_SRCS="$CORE_SRCS $DARWIN_SENDFILE_SRCS" fi

@@ -44,10 +44,12 @@ CORE_SRCS="$CORE_SRCS $FREEBSD_SENDFILE_SRCS" fi -if [ $osreldate -gt 502103 ]; then - echo " + sendfile()'s SF_NODISKIO found" +if [ $NGX_FILE_AIO = YES ]; then + if [ $osreldate -gt 502103 ]; then + echo " + sendfile()'s SF_NODISKIO found" - have=NGX_HAVE_AIO_SENDFILE . auto/have + have=NGX_HAVE_AIO_SENDFILE . auto/have + fi fi # POSIX semaphores @@ -78,7 +80,7 @@ NGX_KQUEUE_CHECKED=YES -# kqueue's NOTE_LAWAT +# kqueue's NOTE_LOWAT if [ $ $version -lt 500000 -a $version -ge 430000 $ \ -o $version -ge 500018 ] @@ -97,25 +99,6 @@ fi -if [ $USE_THREADS = "rfork" ]; then - - echo " + using rfork()" - -# # kqueue's EVFILT_SIGNAL is safe -# -# if [ $version -gt 460101 ]; then -# echo " + kqueue's EVFILT_SIGNAL is safe" -# have=NGX_HAVE_SAFE_EVFILT_SIGNAL . auto/have -# else -# echo "$0: error: the kqueue's EVFILT_SIGNAL is unsafe on this" -# echo "FreeBSD version, so --with-threads=rfork could not be used" -# echo -# -# exit 1 -# fi -fi - - if [ $EVENT_AIO = YES ]; then if [ $ $version -lt 500000 -a $version -ge 430000 $ \ -o $version -ge 500014 ]

@@ -92,14 +92,12 @@ EVENT_DEPS="src/event/ngx_event.h \ src/event/ngx_event_timer.h \ src/event/ngx_event_posted.h \ - src/event/ngx_event_busy_lock.h \ src/event/ngx_event_connect.h \ src/event/ngx_event_pipe.h" EVENT_SRCS="src/event/ngx_event.c \ src/event/ngx_event_timer.c \ src/event/ngx_event_posted.c \ - src/event/ngx_event_busy_lock.c \ src/event/ngx_event_accept.c \ src/event/ngx_event_connect.c \ src/event/ngx_event_pipe.c" @@ -193,14 +191,16 @@ POSIX_DEPS=src/os/unix/ngx_posix_config.h +THREAD_POOL_MODULE=ngx_thread_pool_module +THREAD_POOL_DEPS=src/core/ngx_thread_pool.h +THREAD_POOL_SRCS="src/core/ngx_thread_pool.c + src/os/unix/ngx_thread_cond.c + src/os/unix/ngx_thread_mutex.c + src/os/unix/ngx_thread_id.c" + FREEBSD_DEPS="src/os/unix/ngx_freebsd_config.h src/os/unix/ngx_freebsd.h" FREEBSD_SRCS=src/os/unix/ngx_freebsd_init.c FREEBSD_SENDFILE_SRCS=src/os/unix/ngx_freebsd_sendfile_chain.c -FREEBSD_RFORK_DEPS="src/os/unix/ngx_freebsd_rfork_thread.h" -FREEBSD_RFORK_SRCS="src/os/unix/ngx_freebsd_rfork_thread.c" -FREEBSD_RFORK_THREAD_SRCS="src/os/unix/rfork_thread.S" - -PTHREAD_SRCS="src/os/unix/ngx_pthread_thread.c" LINUX_DEPS="src/os/unix/ngx_linux_config.h src/os/unix/ngx_linux.h" LINUX_SRCS=src/os/unix/ngx_linux_init.c @@ -295,8 +295,7 @@ src/http/ngx_http_variables.h \ src/http/ngx_http_script.h \ src/http/ngx_http_upstream.h \ - src/http/ngx_http_upstream_round_robin.h \ - src/http/ngx_http_busy_lock.h" + src/http/ngx_http_upstream_round_robin.h" HTTP_SRCS="src/http/ngx_http.c \ src/http/ngx_http_core_module.c \ @@ -320,9 +319,6 @@ src/http/modules/ngx_http_headers_filter_module.c \ src/http/modules/ngx_http_not_modified_filter_module.c" -# STUB -HTTP_SRCS="$HTTP_SRCS src/http/ngx_http_busy_lock.c" - HTTP_POSTPONE_FILTER_SRCS=src/http/ngx_http_postpone_filter_module.c HTTP_FILE_CACHE_SRCS=src/http/ngx_http_file_cache.c

@@ -3,34 +3,13 @@ # Copyright (C) Nginx, Inc. -### STUB - -if [ $USE_THREADS != NO ]; then - -cat << END - -$0: error: the threads support is broken now. - -END - exit 1 - fi - -### - - echo echo "Configuration summary" -#case $USE_THREADS in -# rfork) echo " + using rfork()ed threads" ;; -# pthreads) echo " + using libpthread threads library" ;; -# libthr) echo " + using FreeBSD libthr threads library" ;; -# libc_r) echo " + using FreeBSD libc_r threads library" ;; -# linuxthreads) echo " + using FreeBSD LinuxThreads port library" ;; -# NO) echo " + threads are not used" ;; -# *) echo " + using lib$USE_THREADS threads library" ;; -#esac +if [ $USE_THREADS = YES ]; then + echo " + using threads" +fi if [ $USE_PCRE = DISABLED ]; then echo " + PCRE library is disabled"

@@ -0,0 +1,20 @@ + +# Copyright (C) Nginx, Inc. + + +if [ $USE_THREADS = YES ]; then + + if [ "$NGX_PLATFORM" = win32 ]; then + cat << END + +$0: --with-threads is not supported on Windows + +END + exit 1 + fi + + have=NGX_THREADS . auto/have + CORE_DEPS="$CORE_DEPS $THREAD_POOL_DEPS" + CORE_SRCS="$CORE_SRCS $THREAD_POOL_SRCS" + CORE_LIBS="$CORE_LIBS -lpthread" +fi

@@ -50,22 +50,12 @@ case $ngx_size in 4) - if [ "$ngx_type"="long" ]; then - ngx_max_value=2147483647L - else - ngx_max_value=2147483647 - fi - + ngx_max_value=2147483647 ngx_max_len='(sizeof("-2147483648") - 1)' ;; 8) - if [ "$ngx_type"="long long" ]; then - ngx_max_value=9223372036854775807LL - else - ngx_max_value=9223372036854775807L - fi - + ngx_max_value=9223372036854775807LL ngx_max_len='(sizeof("-9223372036854775808") - 1)' ;;

@@ -450,6 +450,29 @@ END exit 1 fi + +else + + ngx_feature="eventfd()" + ngx_feature_name="NGX_HAVE_EVENTFD" + ngx_feature_run=no + ngx_feature_incs="#include <sys/eventfd.h>" + ngx_feature_path= + ngx_feature_libs= + ngx_feature_test="(void) eventfd(0, 0)" + . auto/feature + + if [ $ngx_found = yes ]; then + have=NGX_HAVE_SYS_EVENTFD_H . auto/have + fi + + if [ $ngx_found = no ]; then + + ngx_feature="eventfd() (SYS_eventfd)" + ngx_feature_incs="#include <sys/syscall.h>" + ngx_feature_test="int n = SYS_eventfd" + . auto/feature + fi fi @@ -510,6 +533,7 @@ ngx_type="time_t"; . auto/types/sizeof ngx_param=NGX_TIME_T_SIZE; ngx_value=$ngx_size; . auto/types/value ngx_param=NGX_TIME_T_LEN; ngx_value=$ngx_max_len; . auto/types/value +ngx_param=NGX_MAX_TIME_T_VALUE; ngx_value=$ngx_max_value; . auto/types/value # syscalls, libc calls and some features

@@ -58,6 +58,7 @@ . auto/unix fi +. auto/threads . auto/modules . auto/lib/conf

@@ -139,7 +139,7 @@ 0, NULL }, -#if (NGX_THREADS) +#if (NGX_OLD_THREADS) { ngx_string("worker_threads"), NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_TAKE1, @@ -959,7 +959,7 @@ ccf->user = (ngx_uid_t) NGX_CONF_UNSET_UINT; ccf->group = (ngx_gid_t) NGX_CONF_UNSET_UINT; -#if (NGX_THREADS) +#if (NGX_OLD_THREADS) ccf->worker_threads = NGX_CONF_UNSET; ccf->thread_stack_size = NGX_CONF_UNSET_SIZE; #endif @@ -1000,7 +1000,7 @@ #endif -#if (NGX_THREADS) +#if (NGX_OLD_THREADS) ngx_conf_init_value(ccf->worker_threads, 0); ngx_threads_n = ccf->worker_threads;

@@ -9,8 +9,8 @@ #define _NGINX_H_INCLUDED_ -#define nginx_version 1007009 -#define NGINX_VERSION "1.7.9" +#define nginx_version 1007011 +#define NGINX_VERSION "1.7.11" #define NGINX_VER "nginx/" NGINX_VERSION #ifdef NGX_BUILD

@@ -90,10 +90,21 @@ #endif unsigned need_in_memory:1; unsigned need_in_temp:1; -#if (NGX_HAVE_FILE_AIO) +#if (NGX_HAVE_FILE_AIO || NGX_THREADS) unsigned aio:1; +#endif +#if (NGX_HAVE_FILE_AIO) ngx_output_chain_aio_pt aio_handler; +#if (NGX_HAVE_AIO_SENDFILE) + ssize_t (*aio_preload)(ngx_buf_t *file); +#endif +#endif + +#if (NGX_THREADS) + ngx_int_t (*thread_handler)(ngx_thread_task_t *task, + ngx_file_t *file); + ngx_thread_task_t *thread_task; #endif off_t alignment;

@@ -85,8 +85,11 @@ #if (NGX_PTR_SIZE == 4) #define NGX_INT_T_LEN NGX_INT32_LEN +#define NGX_MAX_INT_T_VALUE 2147483647 + #else #define NGX_INT_T_LEN NGX_INT64_LEN +#define NGX_MAX_INT_T_VALUE 9223372036854775807 #endif

@@ -835,8 +835,6 @@ return NULL; } - /* ngx_mutex_lock */ - c = ngx_cycle->free_connections; if (c == NULL) { @@ -849,16 +847,12 @@ "%ui worker_connections are not enough", ngx_cycle->connection_n); - /* ngx_mutex_unlock */ - return NULL; } ngx_cycle->free_connections = c->data; ngx_cycle->free_connection_n--; - /* ngx_mutex_unlock */ - if (ngx_cycle->files) { ngx_cycle->files[s] = c; } @@ -896,14 +890,10 @@ void ngx_free_connection(ngx_connection_t *c) { - /* ngx_mutex_lock */ - c->data = ngx_cycle->free_connections; ngx_cycle->free_connections = c; ngx_cycle->free_connection_n++; - /* ngx_mutex_unlock */ - if (ngx_cycle->files) { ngx_cycle->files[c->fd] = NULL; } @@ -943,18 +933,6 @@ } } -#if (NGX_THREADS) - - /* - * we have to clean the connection information before the closing - * because another thread may reopen the same file descriptor - * before we clean the connection - */ - - ngx_unlock(&c->lock); - -#endif - if (c->read->posted) { ngx_delete_posted_event(c->read); } @@ -1073,33 +1051,33 @@ struct sockaddr_in6 *sin6; #endif - if (c->local_socklen == 0) { - return NGX_ERROR; - } + addr = 0; - switch (c->local_sockaddr->sa_family) { + if (c->local_socklen) { + switch (c->local_sockaddr->sa_family) { #if (NGX_HAVE_INET6) - case AF_INET6: - sin6 = (struct sockaddr_in6 *) c->local_sockaddr; + case AF_INET6: + sin6 = (struct sockaddr_in6 *) c->local_sockaddr; - for (addr = 0, i = 0; addr == 0 && i < 16; i++) { - addr |= sin6->sin6_addr.s6_addr[i]; - } + for (i = 0; addr == 0 && i < 16; i++) { + addr |= sin6->sin6_addr.s6_addr[i]; + } - break; + break; #endif #if (NGX_HAVE_UNIX_DOMAIN) - case AF_UNIX: - addr = 1; - break; + case AF_UNIX: + addr = 1; + break; #endif - default: /* AF_INET */ - sin = (struct sockaddr_in *) c->local_sockaddr; - addr = sin->sin_addr.s_addr; - break; + default: /* AF_INET */ + sin = (struct sockaddr_in *) c->local_sockaddr; + addr = sin->sin_addr.s_addr; + break; + } } if (addr == 0) {

@@ -181,13 +181,11 @@ #endif #if (NGX_HAVE_AIO_SENDFILE) - unsigned aio_sendfile:1; unsigned busy_count:2; - ngx_buf_t *busy_sendfile; #endif #if (NGX_THREADS) - ngx_atomic_t lock; + ngx_thread_task_t *sendfile_task; #endif };

@@ -22,6 +22,10 @@ typedef struct ngx_event_aio_s ngx_event_aio_t; typedef struct ngx_connection_s ngx_connection_t; +#if (NGX_THREADS) +typedef struct ngx_thread_task_s ngx_thread_task_t; +#endif + typedef void (*ngx_event_handler_pt)(ngx_event_t *ev); typedef void (*ngx_connection_handler_pt)(ngx_connection_t *c);

@@ -26,7 +26,7 @@ ngx_uint_t ngx_test_config; ngx_uint_t ngx_quiet_mode; -#if (NGX_THREADS) +#if (NGX_OLD_THREADS) ngx_tls_key_t ngx_core_tls_key; #endif

@@ -103,7 +103,7 @@ ngx_array_t env; char **environment; -#if (NGX_THREADS) +#if (NGX_OLD_THREADS) ngx_int_t worker_threads; size_t thread_stack_size; #endif @@ -111,10 +111,14 @@ } ngx_core_conf_t; +#if (NGX_OLD_THREADS) + typedef struct { ngx_pool_t *pool; /* pcre's malloc() pool */ } ngx_core_tls_t; +#endif + #define ngx_is_init_cycle(cycle) (cycle->conf_ctx == NULL) @@ -136,7 +140,7 @@ extern ngx_module_t ngx_core_module; extern ngx_uint_t ngx_test_config; extern ngx_uint_t ngx_quiet_mode; -#if (NGX_THREADS) +#if (NGX_OLD_THREADS) extern ngx_tls_key_t ngx_core_tls_key; #endif

@@ -114,7 +114,7 @@ rc = ngx_create_temp_file(&tf->file, tf->path, tf->pool, tf->persistent, tf->clean, tf->access); - if (rc == NGX_ERROR || rc == NGX_AGAIN) { + if (rc != NGX_OK) { return rc; } @@ -1035,10 +1035,18 @@ ctx->access = ngx_de_access(&dir); ctx->mtime = ngx_de_mtime(&dir); - if (ctx->pre_tree_handler(ctx, &file) == NGX_ABORT) { + rc = ctx->pre_tree_handler(ctx, &file); + + if (rc == NGX_ABORT) { goto failed; } + if (rc == NGX_DECLINED) { + ngx_log_debug1(NGX_LOG_DEBUG_CORE, ctx->log, 0, + "tree skip dir \"%s\"", file.data); + continue; + } + if (ngx_walk_tree(ctx, &file) == NGX_ABORT) { goto failed; }

@@ -23,6 +23,12 @@ ngx_log_t *log; +#if (NGX_THREADS) + ngx_int_t (*thread_handler)(ngx_thread_task_t *task, + ngx_file_t *file); + void *thread_ctx; +#endif + #if (NGX_HAVE_FILE_AIO) ngx_event_aio_t *aio; #endif

@@ -312,7 +312,7 @@ continue; } - size--; + size = hinit->max_size; ngx_log_error(NGX_LOG_WARN, hinit->pool->log, 0, "could not build optimal %s, you should increase "

@@ -27,6 +27,10 @@ for (p = text; p < text + len; p++) { + if (octet > 255) { + return INADDR_NONE; + } + c = *p; if (c >= '0' && c <= '9') { @@ -34,7 +38,7 @@ continue; } - if (c == '.' && octet < 256) { + if (c == '.') { addr = (addr << 8) + octet; octet = 0; n++; @@ -44,7 +48,7 @@ return INADDR_NONE; } - if (n == 3 && octet < 256) { + if (n == 3) { addr = (addr << 8) + octet; return htonl(addr); }

@@ -14,6 +14,23 @@ static void ngx_log_insert(ngx_log_t *log, ngx_log_t *new_log); +#if (NGX_DEBUG) + +static void ngx_log_memory_writer(ngx_log_t *log, ngx_uint_t level, + u_char *buf, size_t len); +static void ngx_log_memory_cleanup(void *data); + + +typedef struct { + u_char *start; + u_char *end; + u_char *pos; + ngx_atomic_t written; +} ngx_log_memory_buf_t; + +#endif + + static ngx_command_t ngx_errlog_commands[] = { {ngx_string("error_log"), @@ -91,15 +108,14 @@ va_list args; #endif u_char *p, *last, *msg; - u_char errstr[NGX_MAX_ERROR_STR]; + ssize_t n; ngx_uint_t wrote_stderr, debug_connection; + u_char errstr[NGX_MAX_ERROR_STR]; last = errstr + NGX_MAX_ERROR_STR; - ngx_memcpy(errstr, ngx_cached_err_log_time.data, - ngx_cached_err_log_time.len); - - p = errstr + ngx_cached_err_log_time.len; + p = ngx_cpymem(errstr, ngx_cached_err_log_time.data, + ngx_cached_err_log_time.len); p = ngx_slprintf(p, last, " [%V] ", &err_levels[level]); @@ -150,16 +166,32 @@ if (log->writer) { log->writer(log, level, errstr, p - errstr); - log = log->next; - continue; + goto next; + } + + if (ngx_time() == log->disk_full_time) { + + /* + * on FreeBSD writing to a full filesystem with enabled softupdates + * may block process for much longer time than writing to non-full + * filesystem, so we skip writing to a log for one second + */ + + goto next; } - (void) ngx_write_fd(log->file->fd, errstr, p - errstr); + n = ngx_write_fd(log->file->fd, errstr, p - errstr); + + if (n == -1 && ngx_errno == NGX_ENOSPC) { + log->disk_full_time = ngx_time(); + } if (log->file->fd == ngx_stderr) { wrote_stderr = 1; } + next: + log = log->next; } @@ -231,9 +263,8 @@ u_char errstr[NGX_MAX_ERROR_STR]; last = errstr + NGX_MAX_ERROR_STR; - p = errstr + 7; - ngx_memcpy(errstr, "nginx: ", 7); + p = ngx_cpymem(errstr, "nginx: ", 7); va_start(args, fmt); p = ngx_vslprintf(p, last, fmt, args); @@ -554,6 +585,64 @@ return NGX_CONF_ERROR; } + } else if (ngx_strncmp(value[1].data, "memory:", 7) == 0) { + +#if (NGX_DEBUG) + size_t size, needed; + ngx_pool_cleanup_t *cln; + ngx_log_memory_buf_t *buf; + + value[1].len -= 7; + value[1].data += 7; + + needed = sizeof("MEMLOG :" NGX_LINEFEED) + + cf->conf_file->file.name.len + + NGX_SIZE_T_LEN + + NGX_INT_T_LEN + + NGX_MAX_ERROR_STR; + + size = ngx_parse_size(&value[1]); + + if (size == (size_t) NGX_ERROR || size < needed) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid buffer size \"%V\"", &value[1]); + return NGX_CONF_ERROR; + } + + buf = ngx_palloc(cf->pool, sizeof(ngx_log_memory_buf_t)); + if (buf == NULL) { + return NGX_CONF_ERROR; + } + + buf->start = ngx_pnalloc(cf->pool, size); + if (buf->start == NULL) { + return NGX_CONF_ERROR; + } + + buf->end = buf->start + size; + + buf->pos = ngx_slprintf(buf->start, buf->end, "MEMLOG %uz %V:%ui%N", + size, &cf->conf_file->file.name, + cf->conf_file->line); + + ngx_memset(buf->pos, ' ', buf->end - buf->pos); + + cln = ngx_pool_cleanup_add(cf->pool, 0); + if (cln == NULL) { + return NGX_CONF_ERROR; + } + + cln->data = new_log; + cln->handler = ngx_log_memory_cleanup; + + new_log->writer = ngx_log_memory_writer; + new_log->wdata = buf; + +#else + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "nginx was built without debug support"); + return NGX_CONF_ERROR; +#endif } else if (ngx_strncmp(value[1].data, "syslog:", 7) == 0) { peer = ngx_pcalloc(cf->pool, sizeof(ngx_syslog_peer_t)); @@ -619,3 +708,48 @@ log->next = new_log; } + + +#if (NGX_DEBUG) + +static void +ngx_log_memory_writer(ngx_log_t *log, ngx_uint_t level, u_char *buf, + size_t len) +{ + u_char *p; + size_t avail, written; + ngx_log_memory_buf_t *mem; + + mem = log->wdata; + + if (mem == NULL) { + return; + } + + written = ngx_atomic_fetch_add(&mem->written, len); + + p = mem->pos + written % (mem->end - mem->pos); + + avail = mem->end - p; + + if (avail >= len) { + ngx_memcpy(p, buf, len); + + } else { + ngx_memcpy(p, buf, avail); + ngx_memcpy(mem->pos, buf + avail, len - avail); + } +} + + +static void +ngx_log_memory_cleanup(void *data) +{ + ngx_log_t *log = data; + + ngx_log_debug0(NGX_LOG_DEBUG_CORE, log, 0, "destroy memory log buffer"); + + log->wdata = NULL; +} + +#endif

@@ -53,6 +53,8 @@ ngx_atomic_uint_t connection; + time_t disk_full_time; + ngx_log_handler_pt handler; void *data;

@@ -29,6 +29,10 @@ static ngx_inline ngx_int_t ngx_output_chain_as_is(ngx_output_chain_ctx_t *ctx, ngx_buf_t *buf); +#if (NGX_HAVE_AIO_SENDFILE) +static ngx_int_t ngx_output_chain_aio_setup(ngx_output_chain_ctx_t *ctx, + ngx_file_t *file); +#endif static ngx_int_t ngx_output_chain_add_copy(ngx_pool_t *pool, ngx_chain_t **chain, ngx_chain_t *in); static ngx_int_t ngx_output_chain_align_file_buf(ngx_output_chain_ctx_t *ctx, @@ -45,8 +49,12 @@ ngx_int_t rc, last; ngx_chain_t *cl, *out, **last_out; - if (ctx->in == NULL && ctx->busy == NULL) { - + if (ctx->in == NULL && ctx->busy == NULL +#if (NGX_HAVE_FILE_AIO || NGX_THREADS) + && !ctx->aio +#endif + ) + { /* * the short path for the case when the ctx->in and ctx->busy chains * are empty, the incoming chain is empty too or has the single buf @@ -81,7 +89,7 @@ for ( ;; ) { -#if (NGX_HAVE_FILE_AIO) +#if (NGX_HAVE_FILE_AIO || NGX_THREADS) if (ctx->aio) { return NGX_AGAIN; } @@ -225,6 +233,13 @@ return 1; } +#if (NGX_THREADS) + if (buf->in_file) { + buf->file->thread_handler = ctx->thread_handler; + buf->file->thread_ctx = ctx->filter_ctx; + } +#endif + if (buf->in_file && buf->file->directio) { return 0; } @@ -248,6 +263,12 @@ buf->in_file = 0; } +#if (NGX_HAVE_AIO_SENDFILE) + if (ctx->aio_preload && buf->in_file) { + (void) ngx_output_chain_aio_setup(ctx, buf->file); + } +#endif + if (ctx->need_in_memory && !ngx_buf_in_memory(buf)) { return 0; } @@ -260,6 +281,28 @@ } +#if (NGX_HAVE_AIO_SENDFILE) + +static ngx_int_t +ngx_output_chain_aio_setup(ngx_output_chain_ctx_t *ctx, ngx_file_t *file) +{ + ngx_event_aio_t *aio; + + if (file->aio == NULL && ngx_file_aio_init(file, ctx->pool) != NGX_OK) { + return NGX_ERROR; + } + + aio = file->aio; + + aio->data = ctx->filter_ctx; + aio->preload_handler = ctx->aio_preload; + + return NGX_OK; +} + +#endif + + static ngx_int_t ngx_output_chain_add_copy(ngx_pool_t *pool, ngx_chain_t **chain, ngx_chain_t *in) @@ -523,7 +566,6 @@ #endif #if (NGX_HAVE_FILE_AIO) - if (ctx->aio_handler) { n = ngx_file_aio_read(src->file, dst->pos, (size_t) size, src->file_pos, ctx->pool); @@ -532,15 +574,23 @@ return NGX_AGAIN; } - } else { + } else +#endif +#if (NGX_THREADS) + if (src->file->thread_handler) { + n = ngx_thread_read(&ctx->thread_task, src->file, dst->pos, + (size_t) size, src->file_pos, ctx->pool); + if (n == NGX_AGAIN) { + ctx->aio = 1; + return NGX_AGAIN; + } + + } else +#endif + { n = ngx_read_file(src->file, dst->pos, (size_t) size, src->file_pos); } -#else - - n = ngx_read_file(src->file, dst->pos, (size_t) size, src->file_pos); - -#endif #if (NGX_HAVE_ALIGNED_DIRECTIO) @@ -604,7 +654,7 @@ ngx_chain_writer_ctx_t *ctx = data; off_t size; - ngx_chain_t *cl; + ngx_chain_t *cl, *ln, *chain; ngx_connection_t *c; c = ctx->connection; @@ -613,7 +663,23 @@ #if 1 if (ngx_buf_size(in->buf) == 0 && !ngx_buf_special(in->buf)) { + + ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0, + "zero size buf in chain writer " + "t:%d r:%d f:%d %p %p-%p %p %O-%O", + in->buf->temporary, + in->buf->recycled, + in->buf->in_file, + in->buf->start, + in->buf->pos, + in->buf->last, + in->buf->file, + in->buf->file_pos, + in->buf->file_last); + ngx_debug_point(); + + continue; } #endif @@ -641,9 +707,24 @@ #if 1 if (ngx_buf_size(cl->buf) == 0 && !ngx_buf_special(cl->buf)) { + + ngx_log_error(NGX_LOG_ALERT, ctx->pool->log, 0, + "zero size buf in chain writer " + "t:%d r:%d f:%d %p %p-%p %p %O-%O", + cl->buf->temporary, + cl->buf->recycled, + cl->buf->in_file, + cl->buf->start, + cl->buf->pos, + cl->buf->last, + cl->buf->file, + cl->buf->file_pos, + cl->buf->file_last); + ngx_debug_point(); - } + continue; + } #endif size += ngx_buf_size(cl->buf); @@ -653,15 +734,23 @@ return NGX_OK; } - ctx->out = c->send_chain(c, ctx->out, ctx->limit); + chain = c->send_chain(c, ctx->out, ctx->limit); ngx_log_debug1(NGX_LOG_DEBUG_CORE, c->log, 0, - "chain writer out: %p", ctx->out); + "chain writer out: %p", chain); - if (ctx->out == NGX_CHAIN_ERROR) { + if (chain == NGX_CHAIN_ERROR) { return NGX_ERROR; } + for (cl = ctx->out; cl && cl != chain; /* void */) { + ln = cl; + cl = cl->next; + ngx_free_chain(ctx->pool, ln); + } + + ctx->out = chain; + if (ctx->out == NULL) { ctx->last = &ctx->out;

@@ -12,10 +12,9 @@ ssize_t ngx_parse_size(ngx_str_t *line) { - u_char unit; - size_t len; - ssize_t size; - ngx_int_t scale; + u_char unit; + size_t len; + ssize_t size, scale, max; len = line->len; unit = line->data[len - 1]; @@ -24,21 +23,24 @@ case 'K': case 'k': len--; + max = NGX_MAX_SIZE_T_VALUE / 1024; scale = 1024; break; case 'M': case 'm': len--; + max = NGX_MAX_SIZE_T_VALUE / (1024 * 1024); scale = 1024 * 1024; break; default: + max = NGX_MAX_SIZE_T_VALUE; scale = 1; } size = ngx_atosz(line->data, len); - if (size == NGX_ERROR) { + if (size == NGX_ERROR || size > max) { return NGX_ERROR; } @@ -51,10 +53,9 @@ off_t ngx_parse_offset(ngx_str_t *line) { - u_char unit; - off_t offset; - size_t len; - ngx_int_t scale; + u_char unit; + off_t offset, scale, max; + size_t len; len = line->len; unit = line->data[len - 1]; @@ -63,27 +64,31 @@ case 'K': case 'k': len--; + max = NGX_MAX_OFF_T_VALUE / 1024; scale = 1024; break; case 'M': case 'm': len--; + max = NGX_MAX_OFF_T_VALUE / (1024 * 1024); scale = 1024 * 1024; break; case 'G': case 'g': len--; + max = NGX_MAX_OFF_T_VALUE / (1024 * 1024 * 1024); scale = 1024 * 1024 * 1024; break; default: + max = NGX_MAX_OFF_T_VALUE; scale = 1; } offset = ngx_atoof(line->data, len); - if (offset == NGX_ERROR) { + if (offset == NGX_ERROR || offset > max) { return NGX_ERROR; } @@ -98,7 +103,8 @@ { u_char *p, *last; ngx_int_t value, total, scale; - ngx_uint_t max, valid; + ngx_int_t max, cutoff, cutlim; + ngx_uint_t valid; enum { st_start = 0, st_year, @@ -115,8 +121,9 @@ valid = 0; value = 0; total = 0; + cutoff = NGX_MAX_INT_T_VALUE / 10; + cutlim = NGX_MAX_INT_T_VALUE % 10; step = is_sec ? st_start : st_month; - scale = is_sec ? 1 : 1000; p = line->data; last = p + line->len; @@ -124,6 +131,10 @@ while (p < last) { if (*p >= '0' && *p <= '9') { + if (value >= cutoff && (value > cutoff || *p - '0' > cutlim)) { + return NGX_ERROR; + } + value = value * 10 + (*p++ - '0'); valid = 1; continue; @@ -136,7 +147,7 @@ return NGX_ERROR; } step = st_year; - max = NGX_MAX_INT32_VALUE / (60 * 60 * 24 * 365); + max = NGX_MAX_INT_T_VALUE / (60 * 60 * 24 * 365); scale = 60 * 60 * 24 * 365; break; @@ -145,7 +156,7 @@ return NGX_ERROR; } step = st_month; - max = NGX_MAX_INT32_VALUE / (60 * 60 * 24 * 30); + max = NGX_MAX_INT_T_VALUE / (60 * 60 * 24 * 30); scale = 60 * 60 * 24 * 30; break; @@ -154,7 +165,7 @@ return NGX_ERROR; } step = st_week; - max = NGX_MAX_INT32_VALUE / (60 * 60 * 24 * 7); + max = NGX_MAX_INT_T_VALUE / (60 * 60 * 24 * 7); scale = 60 * 60 * 24 * 7; break; @@ -163,7 +174,7 @@ return NGX_ERROR; } step = st_day; - max = NGX_MAX_INT32_VALUE / (60 * 60 * 24); + max = NGX_MAX_INT_T_VALUE / (60 * 60 * 24); scale = 60 * 60 * 24; break; @@ -172,7 +183,7 @@ return NGX_ERROR; } step = st_hour; - max = NGX_MAX_INT32_VALUE / (60 * 60); + max = NGX_MAX_INT_T_VALUE / (60 * 60); scale = 60 * 60; break; @@ -183,7 +194,7 @@ } p++; step = st_msec; - max = NGX_MAX_INT32_VALUE; + max = NGX_MAX_INT_T_VALUE; scale = 1; break; } @@ -192,7 +203,7 @@ return NGX_ERROR; } step = st_min; - max = NGX_MAX_INT32_VALUE / 60; + max = NGX_MAX_INT_T_VALUE / 60; scale = 60; break; @@ -201,7 +212,7 @@ return NGX_ERROR; } step = st_sec; - max = NGX_MAX_INT32_VALUE; + max = NGX_MAX_INT_T_VALUE; scale = 1; break; @@ -210,7 +221,7 @@ return NGX_ERROR; } step = st_last; - max = NGX_MAX_INT32_VALUE; + max = NGX_MAX_INT_T_VALUE; scale = 1; break; @@ -223,27 +234,40 @@ max /= 1000; } - if ((ngx_uint_t) value > max) { + if (value > max) { return NGX_ERROR; } - total += value * scale; + value *= scale; - if ((ngx_uint_t) total > NGX_MAX_INT32_VALUE) { + if (total > NGX_MAX_INT_T_VALUE - value) { return NGX_ERROR; } + total += value; + value = 0; - scale = is_sec ? 1 : 1000; while (p < last && *p == ' ') { p++; } } - if (valid) { - return total + value * scale; + if (!valid) { + return NGX_ERROR; + } + + if (!is_sec) { + if (value > NGX_MAX_INT_T_VALUE / 1000) { + return NGX_ERROR; + } + + value *= 1000; + } + + if (total > NGX_MAX_INT_T_VALUE - value) { + return NGX_ERROR; } - return NGX_ERROR; + return total + value; }

@@ -80,7 +80,7 @@ static ngx_inline void ngx_regex_malloc_init(ngx_pool_t *pool) { -#if (NGX_THREADS) +#if (NGX_OLD_THREADS) ngx_core_tls_t *tls; if (ngx_threaded) { @@ -98,7 +98,7 @@ static ngx_inline void ngx_regex_malloc_done(void) { -#if (NGX_THREADS) +#if (NGX_OLD_THREADS) ngx_core_tls_t *tls; if (ngx_threaded) { @@ -253,7 +253,7 @@ ngx_regex_malloc(size_t size) { ngx_pool_t *pool; -#if (NGX_THREADS) +#if (NGX_OLD_THREADS) ngx_core_tls_t *tls; if (ngx_threaded) {

@@ -101,6 +101,7 @@ (void) ngx_atomic_fetch_add(mtx->wait, 1); if (*mtx->lock == 0 && ngx_atomic_cmp_set(mtx->lock, 0, ngx_pid)) { + (void) ngx_atomic_fetch_add(mtx->wait, -1); return; } @@ -174,7 +175,7 @@ wait = *mtx->wait; - if (wait == 0) { + if ((ngx_atomic_int_t) wait <= 0) { return; } @@ -258,7 +259,7 @@ #if __osf__ /* Tru64 UNIX */ - if (err == NGX_EACCESS) { + if (err == NGX_EACCES) { return 0; }

@@ -42,7 +42,7 @@ #else -#if (NGX_THREADS) +#if (NGX_OLD_THREADS) #error ngx_spinlock() or ngx_atomic_cmp_set() are not defined !

@@ -429,8 +429,12 @@ case 'N': #if (NGX_WIN32) *buf++ = CR; -#endif + if (buf < last) { + *buf++ = LF; + } +#else *buf++ = LF; +#endif fmt++; continue; @@ -897,26 +901,28 @@ ngx_int_t ngx_atoi(u_char *line, size_t n) { - ngx_int_t value; + ngx_int_t value, cutoff, cutlim; if (n == 0) { return NGX_ERROR; } + cutoff = NGX_MAX_INT_T_VALUE / 10; + cutlim = NGX_MAX_INT_T_VALUE % 10; + for (value = 0; n--; line++) { if (*line < '0' || *line > '9') { return NGX_ERROR; } + if (value >= cutoff && (value > cutoff || *line - '0' > cutlim)) { + return NGX_ERROR; + } + value = value * 10 + (*line - '0'); } - if (value < 0) { - return NGX_ERROR; - - } else { - return value; - } + return value; } @@ -925,13 +931,16 @@ ngx_int_t ngx_atofp(u_char *line, size_t n, size_t point) { - ngx_int_t value; + ngx_int_t value, cutoff, cutlim; ngx_uint_t dot; if (n == 0) { return NGX_ERROR; } + cutoff = NGX_MAX_INT_T_VALUE / 10; + cutlim = NGX_MAX_INT_T_VALUE % 10; + dot = 0; for (value = 0; n--; line++) { @@ -953,98 +962,107 @@ return NGX_ERROR; } + if (value >= cutoff && (value > cutoff || *line - '0' > cutlim)) { + return NGX_ERROR; + } + value = value * 10 + (*line - '0'); point -= dot; } while (point--) { + if (value > cutoff) { + return NGX_ERROR; + } + value = value * 10; } - if (value < 0) { - return NGX_ERROR; - - } else { - return value; - } + return value; } ssize_t ngx_atosz(u_char *line, size_t n) { - ssize_t value; + ssize_t value, cutoff, cutlim; if (n == 0) { return NGX_ERROR; } + cutoff = NGX_MAX_SIZE_T_VALUE / 10; + cutlim = NGX_MAX_SIZE_T_VALUE % 10; + for (value = 0; n--; line++) { if (*line < '0' || *line > '9') { return NGX_ERROR; } + if (value >= cutoff && (value > cutoff || *line - '0' > cutlim)) { + return NGX_ERROR; + } + value = value * 10 + (*line - '0'); } - if (value < 0) { - return NGX_ERROR; - - } else { - return value; - } + return value; } off_t ngx_atoof(u_char *line, size_t n) { - off_t value; + off_t value, cutoff, cutlim; if (n == 0) { return NGX_ERROR; } + cutoff = NGX_MAX_OFF_T_VALUE / 10; + cutlim = NGX_MAX_OFF_T_VALUE % 10; + for (value = 0; n--; line++) { if (*line < '0' || *line > '9') { return NGX_ERROR; } + if (value >= cutoff && (value > cutoff || *line - '0' > cutlim)) { + return NGX_ERROR; + } + value = value * 10 + (*line - '0'); } - if (value < 0) { - return NGX_ERROR; - - } else { - return value; - } + return value; } time_t ngx_atotm(u_char *line, size_t n) { - time_t value; + time_t value, cutoff, cutlim; if (n == 0) { return NGX_ERROR; } + cutoff = NGX_MAX_TIME_T_VALUE / 10; + cutlim = NGX_MAX_TIME_T_VALUE % 10; + for (value = 0; n--; line++) { if (*line < '0' || *line > '9') { return NGX_ERROR; } + if (value >= cutoff && (value > cutoff || *line - '0' > cutlim)) { + return NGX_ERROR; + } + value = value * 10 + (*line - '0'); } - if (value < 0) { - return NGX_ERROR; - - } else { - return value; - } + return value; } @@ -1052,13 +1070,19 @@ ngx_hextoi(u_char *line, size_t n) { u_char c, ch; - ngx_int_t value; + ngx_int_t value, cutoff; if (n == 0) { return NGX_ERROR; } + cutoff = NGX_MAX_INT_T_VALUE / 16; + for (value = 0; n--; line++) { + if (value > cutoff) { + return NGX_ERROR; + } + ch = *line; if (ch >= '0' && ch <= '9') { @@ -1076,12 +1100,7 @@ return NGX_ERROR; } - if (value < 0) { - return NGX_ERROR; - - } else { - return value; - } + return value; }

@@ -0,0 +1,630 @@ + +/* + * Copyright (C) Nginx, Inc. + * Copyright (C) Valentin V. Bartenev + * Copyright (C) Ruslan Ermilov + */ + + +#include <ngx_config.h> +#include <ngx_core.h> +#include <ngx_thread_pool.h> + + +typedef struct { + ngx_array_t pools; +} ngx_thread_pool_conf_t; + + +typedef struct { + ngx_thread_task_t *first; + ngx_thread_task_t **last; +} ngx_thread_pool_queue_t; + +#define ngx_thread_pool_queue_init(q) \ + (q)->first = NULL; \ + (q)->last = &(q)->first + + +struct ngx_thread_pool_s { + ngx_thread_mutex_t mtx; + ngx_thread_pool_queue_t queue; + ngx_int_t waiting; + ngx_thread_cond_t cond; + + ngx_log_t *log; + + ngx_str_t name; + ngx_uint_t threads; + ngx_int_t max_queue; + + u_char *file; + ngx_uint_t line; +}; + + +static ngx_int_t ngx_thread_pool_init(ngx_thread_pool_t *tp, ngx_log_t *log, + ngx_pool_t *pool); +static void ngx_thread_pool_destroy(ngx_thread_pool_t *tp); +static void ngx_thread_pool_exit_handler(void *data, ngx_log_t *log); + +static void *ngx_thread_pool_cycle(void *data); +static void ngx_thread_pool_handler(ngx_event_t *ev); + +static char *ngx_thread_pool(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); + +static void *ngx_thread_pool_create_conf(ngx_cycle_t *cycle); +static char *ngx_thread_pool_init_conf(ngx_cycle_t *cycle, void *conf); + +static ngx_int_t ngx_thread_pool_init_worker(ngx_cycle_t *cycle); +static void ngx_thread_pool_exit_worker(ngx_cycle_t *cycle); + + +static ngx_command_t ngx_thread_pool_commands[] = { + + { ngx_string("thread_pool"), + NGX_MAIN_CONF|NGX_DIRECT_CONF|NGX_CONF_TAKE23, + ngx_thread_pool, + 0, + 0, + NULL }, + + ngx_null_command +}; + + +static ngx_core_module_t ngx_thread_pool_module_ctx = { + ngx_string("thread_pool"), + ngx_thread_pool_create_conf, + ngx_thread_pool_init_conf +}; + + +ngx_module_t ngx_thread_pool_module = { + NGX_MODULE_V1, + &ngx_thread_pool_module_ctx, /* module context */ + ngx_thread_pool_commands, /* module directives */ + NGX_CORE_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + ngx_thread_pool_init_worker, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + ngx_thread_pool_exit_worker, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + + +static ngx_str_t ngx_thread_pool_default = ngx_string("default"); + +static ngx_uint_t ngx_thread_pool_task_id; +static ngx_atomic_t ngx_thread_pool_done_lock; +static ngx_thread_pool_queue_t ngx_thread_pool_done; + + +static ngx_int_t +ngx_thread_pool_init(ngx_thread_pool_t *tp, ngx_log_t *log, ngx_pool_t *pool) +{ + int err; + pthread_t tid; + ngx_uint_t n; + pthread_attr_t attr; + + if (ngx_notify == NULL) { + ngx_log_error(NGX_LOG_ALERT, log, 0, + "the configured event method cannot be used with thread pools"); + return NGX_ERROR; + } + + ngx_thread_pool_queue_init(&tp->queue); + + if (ngx_thread_mutex_create(&tp->mtx, log) != NGX_OK) { + return NGX_ERROR; + } + + if (ngx_thread_cond_create(&tp->cond, log) != NGX_OK) { + (void) ngx_thread_mutex_destroy(&tp->mtx, log); + return NGX_ERROR; + } + + tp->log = log; + + err = pthread_attr_init(&attr); + if (err) { + ngx_log_error(NGX_LOG_ALERT, log, err, + "pthread_attr_init() failed"); + return NGX_ERROR; + } + +#if 0 + err = pthread_attr_setstacksize(&attr, PTHREAD_STACK_MIN); + if (err) { + ngx_log_error(NGX_LOG_ALERT, log, err, + "pthread_attr_setstacksize() failed"); + return NGX_ERROR; + } +#endif + + for (n = 0; n < tp->threads; n++) { + err = pthread_create(&tid, &attr, ngx_thread_pool_cycle, tp); + if (err) { + ngx_log_error(NGX_LOG_ALERT, log, err, + "pthread_create() failed"); + return NGX_ERROR; + } + } + + (void) pthread_attr_destroy(&attr); + + return NGX_OK; +} + + +static void +ngx_thread_pool_destroy(ngx_thread_pool_t *tp) +{ + ngx_uint_t n; + ngx_thread_task_t task; + volatile ngx_uint_t lock; + + ngx_memzero(&task, sizeof(ngx_thread_task_t)); + + task.handler = ngx_thread_pool_exit_handler; + task.ctx = (void *) &lock; + + for (n = 0; n < tp->threads; n++) { + lock = 1; + + if (ngx_thread_task_post(tp, &task) != NGX_OK) { + return; + } + + while (lock) { + ngx_sched_yield(); + } + + task.event.active = 0; + } + + (void) ngx_thread_cond_destroy(&tp->cond, tp->log); + + (void) ngx_thread_mutex_destroy(&tp->mtx, tp->log); +} + + +static void +ngx_thread_pool_exit_handler(void *data, ngx_log_t *log) +{ + ngx_uint_t *lock = data; + + *lock = 0; + + pthread_exit(0); +} + + +ngx_thread_task_t * +ngx_thread_task_alloc(ngx_pool_t *pool, size_t size) +{ + ngx_thread_task_t *task; + + task = ngx_pcalloc(pool, sizeof(ngx_thread_task_t) + size); + if (task == NULL) { + return NULL; + } + + task->ctx = task + 1; + + return task; +} + + +ngx_int_t +ngx_thread_task_post(ngx_thread_pool_t *tp, ngx_thread_task_t *task) +{ + if (task->event.active) { + ngx_log_error(NGX_LOG_ALERT, tp->log, 0, + "task #%ui already active", task->id); + return NGX_ERROR; + } + + if (ngx_thread_mutex_lock(&tp->mtx, tp->log) != NGX_OK) { + return NGX_ERROR; + } + + if (tp->waiting >= tp->max_queue) { + (void) ngx_thread_mutex_unlock(&tp->mtx, tp->log); + + ngx_log_error(NGX_LOG_ERR, tp->log, 0, + "thread pool \"%V\" queue overflow: %i tasks waiting", + &tp->name, tp->waiting); + return NGX_ERROR; + } + + task->event.active = 1; + + task->id = ngx_thread_pool_task_id++; + task->next = NULL; + + if (ngx_thread_cond_signal(&tp->cond, tp->log) != NGX_OK) { + (void) ngx_thread_mutex_unlock(&tp->mtx, tp->log); + return NGX_ERROR; + } + + *tp->queue.last = task; + tp->queue.last = &task->next; + + tp->waiting++; + + (void) ngx_thread_mutex_unlock(&tp->mtx, tp->log); + + ngx_log_debug2(NGX_LOG_DEBUG_CORE, tp->log, 0, + "task #%ui added to thread pool \"%V\"", + task->id, &tp->name); + + return NGX_OK; +} + + +static void * +ngx_thread_pool_cycle(void *data) +{ + ngx_thread_pool_t *tp = data; + + int err; + sigset_t set; + ngx_thread_task_t *task; + +#if 0 + ngx_time_update(); +#endif + + ngx_log_debug1(NGX_LOG_DEBUG_CORE, tp->log, 0, + "thread in pool \"%V\" started", &tp->name); + + sigfillset(&set); + + sigdelset(&set, SIGILL); + sigdelset(&set, SIGFPE); + sigdelset(&set, SIGSEGV); + sigdelset(&set, SIGBUS); + + err = pthread_sigmask(SIG_BLOCK, &set, NULL); + if (err) { + ngx_log_error(NGX_LOG_ALERT, tp->log, err, "pthread_sigmask() failed"); + return NULL; + } + + for ( ;; ) { + if (ngx_thread_mutex_lock(&tp->mtx, tp->log) != NGX_OK) { + return NULL; + } + + /* the number may become negative */ + tp->waiting--; + + while (tp->queue.first == NULL) { + if (ngx_thread_cond_wait(&tp->cond, &tp->mtx, tp->log) + != NGX_OK) + { + (void) ngx_thread_mutex_unlock(&tp->mtx, tp->log); + return NULL; + } + } + + task = tp->queue.first; + tp->queue.first = task->next; + + if (tp->queue.first == NULL) { + tp->queue.last = &tp->queue.first; + } + + if (ngx_thread_mutex_unlock(&tp->mtx, tp->log) != NGX_OK) { + return NULL; + } + +#if 0 + ngx_time_update(); +#endif + + ngx_log_debug2(NGX_LOG_DEBUG_CORE, tp->log, 0, + "run task #%ui in thread pool \"%V\"", + task->id, &tp->name); + + task->handler(task->ctx, tp->log); + + ngx_log_debug2(NGX_LOG_DEBUG_CORE, tp->log, 0, + "complete task #%ui in thread pool \"%V\"", + task->id, &tp->name); + + task->next = NULL; + + ngx_spinlock(&ngx_thread_pool_done_lock, 1, 2048); + + *ngx_thread_pool_done.last = task; + ngx_thread_pool_done.last = &task->next; + + ngx_unlock(&ngx_thread_pool_done_lock); + + (void) ngx_notify(ngx_thread_pool_handler); + } +} + + +static void +ngx_thread_pool_handler(ngx_event_t *ev) +{ + ngx_event_t *event; + ngx_thread_task_t *task; + + ngx_log_debug0(NGX_LOG_DEBUG_CORE, ev->log, 0, "thread pool handler"); + + ngx_spinlock(&ngx_thread_pool_done_lock, 1, 2048); + + task = ngx_thread_pool_done.first; + ngx_thread_pool_done.first = NULL; + ngx_thread_pool_done.last = &ngx_thread_pool_done.first; + + ngx_unlock(&ngx_thread_pool_done_lock); + + while (task) { + ngx_log_debug1(NGX_LOG_DEBUG_CORE, ev->log, 0, + "run completion handler for task #%ui", task->id); + + event = &task->event; + task = task->next; + + event->complete = 1; + event->active = 0; + + event->handler(event); + } +} + + +static void * +ngx_thread_pool_create_conf(ngx_cycle_t *cycle) +{ + ngx_thread_pool_conf_t *tcf; + + tcf = ngx_pcalloc(cycle->pool, sizeof(ngx_thread_pool_conf_t)); + if (tcf == NULL) { + return NULL; + } + + if (ngx_array_init(&tcf->pools, cycle->pool, 4, + sizeof(ngx_thread_pool_t *)) + != NGX_OK) + { + return NULL; + } + + return tcf; +} + + +static char * +ngx_thread_pool_init_conf(ngx_cycle_t *cycle, void *conf) +{ + ngx_thread_pool_conf_t *tcf = conf; + + ngx_uint_t i; + ngx_thread_pool_t **tpp; + + tpp = tcf->pools.elts; + + for (i = 0; i < tcf->pools.nelts; i++) { + + if (tpp[i]->threads) { + continue; + } + + if (tpp[i]->name.len == ngx_thread_pool_default.len + && ngx_strncmp(tpp[i]->name.data, ngx_thread_pool_default.data, + ngx_thread_pool_default.len) + == 0) + { + tpp[i]->threads = 32; + tpp[i]->max_queue = 65536; + continue; + } + + ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, + "unknown thread pool \"%V\" in %s:%ui", + &tpp[i]->name, tpp[i]->file, tpp[i]->line); + + return NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + + +static char * +ngx_thread_pool(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_str_t *value; + ngx_uint_t i; + ngx_thread_pool_t *tp; + + value = cf->args->elts; + + tp = ngx_thread_pool_add(cf, &value[1]); + + if (tp == NULL) { + return NGX_CONF_ERROR; + } + + if (tp->threads) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "duplicate thread pool \"%V\"", &tp->name); + return NGX_CONF_ERROR; + } + + tp->max_queue = 65536; + + for (i = 2; i < cf->args->nelts; i++) { + + if (ngx_strncmp(value[i].data, "threads=", 8) == 0) { + + tp->threads = ngx_atoi(value[i].data + 8, value[i].len - 8); + + if (tp->threads == (ngx_uint_t) NGX_ERROR || tp->threads == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid threads value \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } + + continue; + } + + if (ngx_strncmp(value[i].data, "max_queue=", 10) == 0) { + + tp->max_queue = ngx_atoi(value[i].data + 10, value[i].len - 10); + + if (tp->max_queue == NGX_ERROR) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid max_queue value \"%V\"", &value[i]); + return NGX_CONF_ERROR; + } + + continue; + } + } + + if (tp->threads == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "\"%V\" must have \"threads\" parameter", + &cmd->name); + return NGX_CONF_ERROR; + } + + return NGX_CONF_OK; +} + + +ngx_thread_pool_t * +ngx_thread_pool_add(ngx_conf_t *cf, ngx_str_t *name) +{ + ngx_thread_pool_t *tp, **tpp; + ngx_thread_pool_conf_t *tcf; + + if (name == NULL) { + name = &ngx_thread_pool_default; + } + + tp = ngx_thread_pool_get(cf->cycle, name); + + if (tp) { + return tp; + } + + tp = ngx_pcalloc(cf->pool, sizeof(ngx_thread_pool_t)); + if (tp == NULL) { + return NULL; + } + + tp->name = *name; + tp->file = cf->conf_file->file.name.data; + tp->line = cf->conf_file->line; + + tcf = (ngx_thread_pool_conf_t *) ngx_get_conf(cf->cycle->conf_ctx, + ngx_thread_pool_module); + + tpp = ngx_array_push(&tcf->pools); + if (tpp == NULL) { + return NULL; + } + + *tpp = tp; + + return tp; +} + + +ngx_thread_pool_t * +ngx_thread_pool_get(ngx_cycle_t *cycle, ngx_str_t *name) +{ + ngx_uint_t i; + ngx_thread_pool_t **tpp; + ngx_thread_pool_conf_t *tcf; + + tcf = (ngx_thread_pool_conf_t *) ngx_get_conf(cycle->conf_ctx, + ngx_thread_pool_module); + + tpp = tcf->pools.elts; + + for (i = 0; i < tcf->pools.nelts; i++) { + + if (tpp[i]->name.len == name->len + && ngx_strncmp(tpp[i]->name.data, name->data, name->len) == 0) + { + return tpp[i]; + } + } + + return NULL; +} + + +static ngx_int_t +ngx_thread_pool_init_worker(ngx_cycle_t *cycle) +{ + ngx_uint_t i; + ngx_thread_pool_t **tpp; + ngx_thread_pool_conf_t *tcf; + + if (ngx_process != NGX_PROCESS_WORKER + && ngx_process != NGX_PROCESS_SINGLE) + { + return NGX_OK; + } + + tcf = (ngx_thread_pool_conf_t *) ngx_get_conf(cycle->conf_ctx, + ngx_thread_pool_module); + + if (tcf == NULL) { + return NGX_OK; + } + + ngx_thread_pool_queue_init(&ngx_thread_pool_done); + + tpp = tcf->pools.elts; + + for (i = 0; i < tcf->pools.nelts; i++) { + if (ngx_thread_pool_init(tpp[i], cycle->log, cycle->pool) != NGX_OK) { + return NGX_ERROR; + } + } + + return NGX_OK; +} + + +static void +ngx_thread_pool_exit_worker(ngx_cycle_t *cycle) +{ + ngx_uint_t i; + ngx_thread_pool_t **tpp; + ngx_thread_pool_conf_t *tcf; + + if (ngx_process != NGX_PROCESS_WORKER + && ngx_process != NGX_PROCESS_SINGLE) + { + return; + } + + tcf = (ngx_thread_pool_conf_t *) ngx_get_conf(cycle->conf_ctx, + ngx_thread_pool_module); + + if (tcf == NULL) { + return; + } + + tpp = tcf->pools.elts; + + for (i = 0; i < tcf->pools.nelts; i++) { + ngx_thread_pool_destroy(tpp[i]); + } +}

@@ -0,0 +1,36 @@ + +/* + * Copyright (C) Nginx, Inc. + * Copyright (C) Valentin V. Bartenev + */ + + +#ifndef _NGX_THREAD_POOL_H_INCLUDED_ +#define _NGX_THREAD_POOL_H_INCLUDED_ + + +#include <ngx_config.h> +#include <ngx_core.h> +#include <ngx_event.h> + + +struct ngx_thread_task_s { + ngx_thread_task_t *next; + ngx_uint_t id; + void *ctx; + void (*handler)(void *data, ngx_log_t *log); + ngx_event_t event; +}; + + +typedef struct ngx_thread_pool_s ngx_thread_pool_t; + + +ngx_thread_pool_t *ngx_thread_pool_add(ngx_conf_t *cf, ngx_str_t *name); +ngx_thread_pool_t *ngx_thread_pool_get(ngx_cycle_t *cycle, ngx_str_t *name); + +ngx_thread_task_t *ngx_thread_task_alloc(ngx_pool_t *pool, size_t size); +ngx_int_t ngx_thread_task_post(ngx_thread_pool_t *tp, ngx_thread_task_t *task); + + +#endif /* _NGX_THREAD_POOL_H_INCLUDED_ */

@@ -48,7 +48,7 @@ NULL, /* disable an event */ NULL, /* add an connection */ ngx_aio_del_connection, /* delete an connection */ - NULL, /* process the changes */ + NULL, /* trigger a notify */ ngx_aio_process_events, /* process the events */ ngx_aio_init, /* init the events */ ngx_aio_done /* done the events */

@@ -88,7 +88,7 @@ ngx_devpoll_del_event, /* disable an event */ NULL, /* add an connection */ NULL, /* delete an connection */ - NULL, /* process the changes */ + NULL, /* trigger a notify */ ngx_devpoll_process_events, /* process the events */ ngx_devpoll_init, /* init the events */ ngx_devpoll_done, /* done the events */

@@ -70,12 +70,15 @@ return -1; } +#if (NGX_HAVE_EVENTFD) +#define SYS_eventfd 323 +#endif + #if (NGX_HAVE_FILE_AIO) #define SYS_io_setup 245 #define SYS_io_destroy 246 #define SYS_io_getevents 247 -#define SYS_eventfd 323 typedef u_int aio_context_t; @@ -88,7 +91,7 @@ #endif -#endif +#endif /* NGX_TEST_BUILD_EPOLL */ typedef struct { @@ -98,6 +101,10 @@ static ngx_int_t ngx_epoll_init(ngx_cycle_t *cycle, ngx_msec_t timer); +#if (NGX_HAVE_EVENTFD) +static ngx_int_t ngx_epoll_notify_init(ngx_log_t *log); +static void ngx_epoll_notify_handler(ngx_event_t *ev); +#endif static void ngx_epoll_done(ngx_cycle_t *cycle); static ngx_int_t ngx_epoll_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags); @@ -106,6 +113,9 @@ static ngx_int_t ngx_epoll_add_connection(ngx_connection_t *c); static ngx_int_t ngx_epoll_del_connection(ngx_connection_t *c, ngx_uint_t flags); +#if (NGX_HAVE_EVENTFD) +static ngx_int_t ngx_epoll_notify(ngx_event_handler_pt handler); +#endif static ngx_int_t ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags); @@ -120,6 +130,12 @@ static struct epoll_event *event_list; static ngx_uint_t nevents; +#if (NGX_HAVE_EVENTFD) +static int notify_fd = -1; +static ngx_event_t notify_event; +static ngx_connection_t notify_conn; +#endif + #if (NGX_HAVE_FILE_AIO) int ngx_eventfd = -1; @@ -164,7 +180,11 @@ ngx_epoll_del_event, /* disable an event */ ngx_epoll_add_connection, /* add an connection */ ngx_epoll_del_connection, /* delete an connection */ - NULL, /* process the changes */ +#if (NGX_HAVE_EVENTFD) + ngx_epoll_notify, /* trigger a notify */ +#else + NULL, /* trigger a notify */ +#endif ngx_epoll_process_events, /* process the events */ ngx_epoll_init, /* init the events */ ngx_epoll_done, /* done the events */ @@ -307,6 +327,12 @@ return NGX_ERROR; } +#if (NGX_HAVE_EVENTFD) + if (ngx_epoll_notify_init(cycle->log) != NGX_OK) { + return NGX_ERROR; + } +#endif + #if (NGX_HAVE_FILE_AIO) ngx_epoll_aio_init(cycle, epcf); @@ -344,6 +370,85 @@ } +#if (NGX_HAVE_EVENTFD) + +static ngx_int_t +ngx_epoll_notify_init(ngx_log_t *log) +{ + struct epoll_event ee; + +#if (NGX_HAVE_SYS_EVENTFD_H) + notify_fd = eventfd(0, 0); +#else + notify_fd = syscall(SYS_eventfd, 0); +#endif + + if (notify_fd == -1) { + ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, "eventfd() failed"); + return NGX_ERROR; + } + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, log, 0, + "notify eventfd: %d", notify_fd); + + notify_event.handler = ngx_epoll_notify_handler; + notify_event.log = log; + notify_event.active = 1; + + notify_conn.fd = notify_fd; + notify_conn.read = &notify_event; + notify_conn.log = log; + + ee.events = EPOLLIN|EPOLLET; + ee.data.ptr = &notify_conn; + + if (epoll_ctl(ep, EPOLL_CTL_ADD, notify_fd, &ee) == -1) { + ngx_log_error(NGX_LOG_EMERG, log, ngx_errno, + "epoll_ctl(EPOLL_CTL_ADD, eventfd) failed"); + + if (close(notify_fd) == -1) { + ngx_log_error(NGX_LOG_ALERT, log, ngx_errno, + "eventfd close() failed"); + } + + return NGX_ERROR; + } + + return NGX_OK; +} + + +static void +ngx_epoll_notify_handler(ngx_event_t *ev) +{ + ssize_t n; + uint64_t count; + ngx_err_t err; + ngx_event_handler_pt handler; + + if (++ev->index == NGX_MAX_UINT32_VALUE) { + ev->index = 0; + + n = read(notify_fd, &count, sizeof(uint64_t)); + + err = ngx_errno; + + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ev->log, 0, + "read() eventfd %d: %z count:%uL", notify_fd, n, count); + + if ((size_t) n != sizeof(uint64_t)) { + ngx_log_error(NGX_LOG_ALERT, ev->log, err, + "read() eventfd %d failed", notify_fd); + } + } + + handler = ev->data; + handler(ev); +} + +#endif + + static void ngx_epoll_done(ngx_cycle_t *cycle) { @@ -354,6 +459,17 @@ ep = -1; +#if (NGX_HAVE_EVENTFD) + + if (close(notify_fd) == -1) { + ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, + "eventfd close() failed"); + } + + notify_fd = -1; + +#endif + #if (NGX_HAVE_FILE_AIO) if (ngx_eventfd != -1) { @@ -560,6 +676,27 @@ } +#if (NGX_HAVE_EVENTFD) + +static ngx_int_t +ngx_epoll_notify(ngx_event_handler_pt handler) +{ + static uint64_t inc = 1; + + if ((size_t) write(notify_fd, &inc, sizeof(uint64_t)) != sizeof(uint64_t)) { + ngx_log_error(NGX_LOG_ALERT, notify_event.log, ngx_errno, + "write() to eventfd %d failed", notify_fd); + return NGX_ERROR; + } + + notify_event.data = handler; + + return NGX_OK; +} + +#endif + + static ngx_int_t ngx_epoll_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) {

@@ -93,6 +93,13 @@ return -1; } +int port_send(int port, int events, void *user); + +int port_send(int port, int events, void *user) +{ + return -1; +} + int timer_create(clockid_t clock_id, struct sigevent *evp, timer_t *timerid); @@ -133,6 +140,7 @@ ngx_uint_t flags); static ngx_int_t ngx_eventport_del_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags); +static ngx_int_t ngx_eventport_notify(ngx_event_handler_pt handler); static ngx_int_t ngx_eventport_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags); @@ -143,6 +151,7 @@ static port_event_t *event_list; static ngx_uint_t nevents; static timer_t event_timer = (timer_t) -1; +static ngx_event_t notify_event; static ngx_str_t eventport_name = ngx_string("eventport"); @@ -172,7 +181,7 @@ ngx_eventport_del_event, /* disable an event */ NULL, /* add an connection */ NULL, /* delete an connection */ - NULL, /* process the changes */ + ngx_eventport_notify, /* trigger a notify */ ngx_eventport_process_events, /* process the events */ ngx_eventport_init, /* init the events */ ngx_eventport_done, /* done the events */ @@ -214,6 +223,9 @@ "port_create() failed"); return NGX_ERROR; } + + notify_event.active = 1; + notify_event.log = cycle->log; } if (nevents < epcf->events) { @@ -405,6 +417,21 @@ } +static ngx_int_t +ngx_eventport_notify(ngx_event_handler_pt handler) +{ + notify_event.handler = handler; + + if (port_send(ep, 0, &notify_event) != 0) { + ngx_log_error(NGX_LOG_ALERT, notify_event.log, ngx_errno, + "port_send() failed"); + return NGX_ERROR; + } + + return NGX_OK; +} + + ngx_int_t ngx_eventport_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) @@ -579,9 +606,15 @@ continue; + case PORT_SOURCE_USER: + + ev->handler(ev); + + continue; + default: ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, - "unexpected even_port object %d", + "unexpected eventport object %d", event_list[i].portev_object); continue; }

@@ -17,6 +17,9 @@ static ngx_int_t ngx_kqueue_init(ngx_cycle_t *cycle, ngx_msec_t timer); +#ifdef EVFILT_USER +static ngx_int_t ngx_kqueue_notify_init(ngx_log_t *log); +#endif static void ngx_kqueue_done(ngx_cycle_t *cycle); static ngx_int_t ngx_kqueue_add_event(ngx_event_t *ev, ngx_int_t event, ngx_uint_t flags); @@ -24,7 +27,9 @@ ngx_uint_t flags); static ngx_int_t ngx_kqueue_set_event(ngx_event_t *ev, ngx_int_t filter, ngx_uint_t flags); -static ngx_int_t ngx_kqueue_process_changes(ngx_cycle_t *cycle, ngx_uint_t try); +#ifdef EVFILT_USER +static ngx_int_t ngx_kqueue_notify(ngx_event_handler_pt handler); +#endif static ngx_int_t ngx_kqueue_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags); static ngx_inline void ngx_kqueue_dump_event(ngx_log_t *log, @@ -36,25 +41,16 @@ int ngx_kqueue = -1; -/* - * The "change_list" should be declared as ngx_thread_volatile. - * However, the use of the change_list is localized in kqueue functions and - * is protected by the mutex so even the "icc -ipo" should not build the code - * with the race condition. Thus we avoid the declaration to make a more - * readable code. - */ - -static struct kevent *change_list, *change_list0, *change_list1; +static struct kevent *change_list; static struct kevent *event_list; static ngx_uint_t max_changes, nchanges, nevents; -#if (NGX_THREADS) -static ngx_mutex_t *list_mutex; -static ngx_mutex_t *kevent_mutex; +#ifdef EVFILT_USER +static ngx_event_t notify_event; +static struct kevent notify_kev; #endif - static ngx_str_t kqueue_name = ngx_string("kqueue"); static ngx_command_t ngx_kqueue_commands[] = { @@ -89,7 +85,11 @@ ngx_kqueue_del_event, /* disable an event */ NULL, /* add an connection */ NULL, /* delete an connection */ - ngx_kqueue_process_changes, /* process the changes */ +#ifdef EVFILT_USER + ngx_kqueue_notify, /* trigger a notify */ +#else + NULL, /* trigger a notify */ +#endif ngx_kqueue_process_events, /* process the events */ ngx_kqueue_init, /* init the events */ ngx_kqueue_done /* done the events */ @@ -133,18 +133,10 @@ return NGX_ERROR; } -#if (NGX_THREADS) - - list_mutex = ngx_mutex_init(cycle->log, 0); - if (list_mutex == NULL) { - return NGX_ERROR; - } - - kevent_mutex = ngx_mutex_init(cycle->log, 0); - if (kevent_mutex == NULL) { +#ifdef EVFILT_USER + if (ngx_kqueue_notify_init(cycle->log) != NGX_OK) { return NGX_ERROR; } - #endif } @@ -163,27 +155,15 @@ nchanges = 0; } - if (change_list0) { - ngx_free(change_list0); - } - - change_list0 = ngx_alloc(kcf->changes * sizeof(struct kevent), - cycle->log); - if (change_list0 == NULL) { - return NGX_ERROR; - } - - if (change_list1) { - ngx_free(change_list1); + if (change_list) { + ngx_free(change_list); } - change_list1 = ngx_alloc(kcf->changes * sizeof(struct kevent), - cycle->log); - if (change_list1 == NULL) { + change_list = ngx_alloc(kcf->changes * sizeof(struct kevent), + cycle->log); + if (change_list == NULL) { return NGX_ERROR; } - - change_list = change_list0; } max_changes = kcf->changes; @@ -247,6 +227,37 @@ } +#ifdef EVFILT_USER + +static ngx_int_t +ngx_kqueue_notify_init(ngx_log_t *log) +{ + notify_kev.ident = 0; + notify_kev.filter = EVFILT_USER; + notify_kev.data = 0; + notify_kev.flags = EV_ADD|EV_CLEAR; + notify_kev.fflags = 0; + notify_kev.udata = 0; + + if (kevent(ngx_kqueue, &notify_kev, 1, NULL, 0, NULL) == -1) { + ngx_log_error(NGX_LOG_ALERT, log, ngx_errno, + "kevent(EVFILT_USER, EV_ADD) failed"); + return NGX_ERROR; + } + + notify_event.active = 1; + notify_event.log = log; + + notify_kev.flags = 0; + notify_kev.fflags = NOTE_TRIGGER; + notify_kev.udata = NGX_KQUEUE_UDATA_T ((uintptr_t) &notify_event); + + return NGX_OK; +} + +#endif + + static void ngx_kqueue_done(ngx_cycle_t *cycle) { @@ -257,17 +268,9 @@ ngx_kqueue = -1; -#if (NGX_THREADS) - ngx_mutex_destroy(kevent_mutex); - ngx_mutex_destroy(list_mutex); -#endif - - ngx_free(change_list1); - ngx_free(change_list0); + ngx_free(change_list); ngx_free(event_list); - change_list1 = NULL; - change_list0 = NULL; change_list = NULL; event_list = NULL; max_changes = 0; @@ -289,8 +292,6 @@ ev->disabled = 0; ev->oneshot = (flags & NGX_ONESHOT_EVENT) ? 1 : 0; - ngx_mutex_lock(list_mutex); - #if 0 if (ev->index < nchanges @@ -315,8 +316,6 @@ e->index = ev->index; } - ngx_mutex_unlock(list_mutex); - return NGX_OK; } @@ -325,8 +324,6 @@ ngx_log_error(NGX_LOG_ALERT, ev->log, 0, "previous event on #%d were not passed in kernel", c->fd); - ngx_mutex_unlock(list_mutex); - return NGX_ERROR; } @@ -334,8 +331,6 @@ rc = ngx_kqueue_set_event(ev, event, EV_ADD|EV_ENABLE|flags); - ngx_mutex_unlock(list_mutex); - return rc; } @@ -349,8 +344,6 @@ ev->active = 0; ev->disabled = 0; - ngx_mutex_lock(list_mutex); - if (ev->index < nchanges && ((uintptr_t) change_list[ev->index].udata & (uintptr_t) ~1) == (uintptr_t) ev) @@ -370,8 +363,6 @@ e->index = ev->index; } - ngx_mutex_unlock(list_mutex); - return NGX_OK; } @@ -382,7 +373,6 @@ */ if (flags & NGX_CLOSE_EVENT) { - ngx_mutex_unlock(list_mutex); return NGX_OK; } @@ -395,8 +385,6 @@ rc = ngx_kqueue_set_event(ev, event, flags); - ngx_mutex_unlock(list_mutex); - return rc; } @@ -487,6 +475,25 @@ } +#ifdef EVFILT_USER + +static ngx_int_t +ngx_kqueue_notify(ngx_event_handler_pt handler) +{ + notify_event.handler = handler; + + if (kevent(ngx_kqueue, &notify_kev, 1, NULL, 0, NULL) == -1) { + ngx_log_error(NGX_LOG_ALERT, notify_event.log, ngx_errno, + "kevent(EVFILT_USER, NOTE_TRIGGER) failed"); + return NGX_ERROR; + } + + return NGX_OK; +} + +#endif + + static ngx_int_t ngx_kqueue_process_events(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags) @@ -499,17 +506,8 @@ ngx_queue_t *queue; struct timespec ts, *tp; - if (ngx_threaded) { - if (ngx_kqueue_process_changes(cycle, 0) == NGX_ERROR) { - return NGX_ERROR; - } - - n = 0; - - } else { - n = (int) nchanges; - nchanges = 0; - } + n = (int) nchanges; + nchanges = 0; if (timer == NGX_TIMER_INFINITE) { tp = NULL; @@ -647,6 +645,11 @@ break; +#ifdef EVFILT_USER + case EVFILT_USER: + break; +#endif + default: ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, "unexpected kevent() filter %d", @@ -670,59 +673,6 @@ } -static ngx_int_t -ngx_kqueue_process_changes(ngx_cycle_t *cycle, ngx_uint_t try) -{ - int n; - ngx_int_t rc; - ngx_err_t err; - struct timespec ts; - struct kevent *changes; - - ngx_mutex_lock(kevent_mutex); - - ngx_mutex_lock(list_mutex); - - if (nchanges == 0) { - ngx_mutex_unlock(list_mutex); - ngx_mutex_unlock(kevent_mutex); - return NGX_OK; - } - - changes = change_list; - if (change_list == change_list0) { - change_list = change_list1; - } else { - change_list = change_list0; - } - - n = (int) nchanges; - nchanges = 0; - - ngx_mutex_unlock(list_mutex); - - ts.tv_sec = 0; - ts.tv_nsec = 0; - - ngx_log_debug1(NGX_LOG_DEBUG_EVENT, cycle->log, 0, - "kevent changes: %d", n); - - if (kevent(ngx_kqueue, changes, n, NULL, 0, &ts) == -1) { - err = ngx_errno; - ngx_log_error((err == NGX_EINTR) ? NGX_LOG_INFO : NGX_LOG_ALERT, - cycle->log, err, "kevent() failed"); - rc = NGX_ERROR; - - } else { - rc = NGX_OK; - } - - ngx_mutex_unlock(kevent_mutex); - - return rc; -} - - static ngx_inline void ngx_kqueue_dump_event(ngx_log_t *log, struct kevent *kev) {

@@ -39,7 +39,7 @@ ngx_poll_del_event, /* disable an event */ NULL, /* add an connection */ NULL, /* delete an connection */ - NULL, /* process the changes */ + NULL, /* trigger a notify */ ngx_poll_process_events, /* process the events */ ngx_poll_init, /* init the events */ ngx_poll_done /* done the events */ @@ -413,7 +413,7 @@ return NGX_CONF_OK; } -#if (NGX_THREADS) +#if (NGX_OLD_THREADS) ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, "poll() is not supported in the threaded mode");

@@ -130,7 +130,7 @@ NULL, /* disable an event */ ngx_rtsig_add_connection, /* add an connection */ ngx_rtsig_del_connection, /* delete an connection */ - NULL, /* process the changes */ + NULL, /* trigger a notify */ ngx_rtsig_process_events, /* process the events */ ngx_rtsig_init, /* init the events */ ngx_rtsig_done, /* done the events */

@@ -47,7 +47,7 @@ ngx_select_del_event, /* disable an event */ NULL, /* add an connection */ NULL, /* delete an connection */ - NULL, /* process the changes */ + NULL, /* trigger a notify */ ngx_select_process_events, /* process the events */ ngx_select_init, /* init the events */ ngx_select_done /* done the events */ @@ -419,7 +419,7 @@ return NGX_CONF_ERROR; } -#if (NGX_THREADS) +#if (NGX_OLD_THREADS) ngx_log_error(NGX_LOG_EMERG, cycle->log, 0, "select() is not supported in the threaded mode");

@@ -48,7 +48,7 @@ ngx_select_del_event, /* disable an event */ NULL, /* add an connection */ NULL, /* delete an connection */ - NULL, /* process the changes */ + NULL, /* trigger a notify */ ngx_select_process_events, /* process the events */ ngx_select_init, /* init the events */ ngx_select_done /* done the events */

@@ -212,7 +212,7 @@ timer = ngx_event_find_timer(); flags = NGX_UPDATE_TIME; -#if (NGX_THREADS) +#if (NGX_OLD_THREADS) if (timer == NGX_TIMER_INFINITE || timer > 500) { timer = 500; @@ -721,10 +721,6 @@ c[i].fd = (ngx_socket_t) -1; next = &c[i]; - -#if (NGX_THREADS) - c[i].lock = 0; -#endif } while (i); cycle->free_connections = next;

@@ -27,14 +27,6 @@ #endif -typedef struct { - ngx_uint_t lock; - - ngx_event_t *events; - ngx_event_t *last; -} ngx_event_mutex_t; - - struct ngx_event_s { void *data; @@ -168,23 +160,21 @@ ngx_event_handler_pt handler; ngx_file_t *file; +#if (NGX_HAVE_AIO_SENDFILE) + ssize_t (*preload_handler)(ngx_buf_t *file); +#endif + ngx_fd_t fd; #if (NGX_HAVE_EVENTFD) int64_t res; -#if (NGX_TEST_BUILD_EPOLL) - ngx_err_t err; - size_t nbytes; #endif -#else + +#if !(NGX_HAVE_EVENTFD) || (NGX_TEST_BUILD_EPOLL) ngx_err_t err; size_t nbytes; #endif -#if (NGX_HAVE_AIO_SENDFILE) - off_t last_offset; -#endif - ngx_aiocb_t aiocb; ngx_event_t event; }; @@ -202,7 +192,8 @@ ngx_int_t (*add_conn)(ngx_connection_t *c); ngx_int_t (*del_conn)(ngx_connection_t *c, ngx_uint_t flags); - ngx_int_t (*process_changes)(ngx_cycle_t *cycle, ngx_uint_t nowait); + ngx_int_t (*notify)(ngx_event_handler_pt handler); + ngx_int_t (*process_events)(ngx_cycle_t *cycle, ngx_msec_t timer, ngx_uint_t flags); @@ -415,7 +406,6 @@ #endif -#define ngx_process_changes ngx_event_actions.process_changes #define ngx_process_events ngx_event_actions.process_events #define ngx_done_events ngx_event_actions.done @@ -424,6 +414,8 @@ #define ngx_add_conn ngx_event_actions.add_conn #define ngx_del_conn ngx_event_actions.del_conn +#define ngx_notify ngx_event_actions.notify + #define ngx_add_timer ngx_event_add_timer #define ngx_del_timer ngx_event_del_timer @@ -533,7 +525,6 @@ #include <ngx_event_timer.h> #include <ngx_event_posted.h> -#include <ngx_event_busy_lock.h> #if (NGX_WIN32) #include <ngx_iocp_module.h>

@@ -53,10 +53,6 @@ ngx_event_save_peer_session_pt save_session; #endif -#if (NGX_THREADS) - ngx_atomic_t *lock; -#endif - ngx_addr_t *local; int rcvbuf;

@@ -249,6 +249,12 @@ SSL_CTX_set_options(ssl->ctx, SSL_OP_SINGLE_DH_USE); +#ifdef SSL_CTRL_CLEAR_OPTIONS + /* only in 0.9.8m+ */ + SSL_CTX_clear_options(ssl->ctx, + SSL_OP_NO_SSLv2|SSL_OP_NO_SSLv3|SSL_OP_NO_TLSv1); +#endif + if (!(protocols & NGX_SSL_SSLv2)) { SSL_CTX_set_options(ssl->ctx, SSL_OP_NO_SSLv2); } @@ -259,11 +265,13 @@ SSL_CTX_set_options(ssl->ctx, SSL_OP_NO_TLSv1); } #ifdef SSL_OP_NO_TLSv1_1 + SSL_CTX_clear_options(ssl->ctx, SSL_OP_NO_TLSv1_1); if (!(protocols & NGX_SSL_TLSv1_1)) { SSL_CTX_set_options(ssl->ctx, SSL_OP_NO_TLSv1_1); } #endif #ifdef SSL_OP_NO_TLSv1_2 + SSL_CTX_clear_options(ssl->ctx, SSL_OP_NO_TLSv1_2); if (!(protocols & NGX_SSL_TLSv1_2)) { SSL_CTX_set_options(ssl->ctx, SSL_OP_NO_TLSv1_2); } @@ -277,6 +285,10 @@ SSL_CTX_set_mode(ssl->ctx, SSL_MODE_RELEASE_BUFFERS); #endif +#ifdef SSL_MODE_NO_AUTO_CHAIN + SSL_CTX_set_mode(ssl->ctx, SSL_MODE_NO_AUTO_CHAIN); +#endif + SSL_CTX_set_read_ahead(ssl->ctx, 1); SSL_CTX_set_info_callback(ssl->ctx, ngx_ssl_info_callback); @@ -1516,7 +1528,6 @@ } in->buf->pos += n; - c->sent += n; if (in->buf->pos == in->buf->last) { in = in->next; @@ -1617,7 +1628,6 @@ } buf->pos += n; - c->sent += n; if (n < size) { break; @@ -1675,6 +1685,8 @@ ngx_post_event(c->read, &ngx_posted_events); } + c->sent += n; + return n; }

@@ -376,7 +376,7 @@ ngx_log_debug8(NGX_LOG_DEBUG_EVENT, p->log, 0, "pipe buf busy s:%d t:%d f:%d " "%p, pos %p, size: %z " - "file: %O, size: %z", + "file: %O, size: %O", (cl->buf->shadow ? 1 : 0), cl->buf->temporary, cl->buf->in_file, cl->buf->start, cl->buf->pos, @@ -389,7 +389,7 @@ ngx_log_debug8(NGX_LOG_DEBUG_EVENT, p->log, 0, "pipe buf out s:%d t:%d f:%d " "%p, pos %p, size: %z " - "file: %O, size: %z", + "file: %O, size: %O", (cl->buf->shadow ? 1 : 0), cl->buf->temporary, cl->buf->in_file, cl->buf->start, cl->buf->pos, @@ -402,7 +402,7 @@ ngx_log_debug8(NGX_LOG_DEBUG_EVENT, p->log, 0, "pipe buf in s:%d t:%d f:%d " "%p, pos %p, size: %z " - "file: %O, size: %z", + "file: %O, size: %O", (cl->buf->shadow ? 1 : 0), cl->buf->temporary, cl->buf->in_file, cl->buf->start, cl->buf->pos, @@ -415,7 +415,7 @@ ngx_log_debug8(NGX_LOG_DEBUG_EVENT, p->log, 0, "pipe buf free s:%d t:%d f:%d " "%p, pos %p, size: %z " - "file: %O, size: %z", + "file: %O, size: %O", (cl->buf->shadow ? 1 : 0), cl->buf->temporary, cl->buf->in_file, cl->buf->start, cl->buf->pos,

@@ -81,8 +81,12 @@ size_t length; size_t padding; + ngx_chain_t *free; + ngx_chain_t *busy; + unsigned fastcgi_stdout:1; unsigned large_stderr:1; + unsigned header_sent:1; ngx_array_t *split_parts; @@ -147,6 +151,8 @@ #endif static ngx_int_t ngx_http_fastcgi_create_request(ngx_http_request_t *r); static ngx_int_t ngx_http_fastcgi_reinit_request(ngx_http_request_t *r); +static ngx_int_t ngx_http_fastcgi_body_output_filter(void *data, + ngx_chain_t *in); static ngx_int_t ngx_http_fastcgi_process_header(ngx_http_request_t *r); static ngx_int_t ngx_http_fastcgi_input_filter_init(void *data); static ngx_int_t ngx_http_fastcgi_input_filter(ngx_event_pipe_t *p, @@ -257,6 +263,13 @@ offsetof(ngx_http_fastcgi_loc_conf_t, upstream.buffering), NULL }, + { ngx_string("fastcgi_request_buffering"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_fastcgi_loc_conf_t, upstream.request_buffering), + NULL }, + { ngx_string("fastcgi_ignore_client_abort"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, ngx_conf_set_flag_slot, @@ -703,6 +716,12 @@ u->input_filter = ngx_http_fastcgi_non_buffered_filter; u->input_filter_ctx = r; + if (!flcf->upstream.request_buffering + && flcf->upstream.pass_request_body) + { + r->request_body_no_buffering = 1; + } + rc = ngx_http_read_client_request_body(r, ngx_http_upstream_init); if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { @@ -799,6 +818,7 @@ ngx_chain_t *cl, *body; ngx_list_part_t *part; ngx_table_elt_t *header, **ignored; + ngx_http_upstream_t *u; ngx_http_script_code_pt code; ngx_http_script_engine_t e, le; ngx_http_fastcgi_header_t *h; @@ -810,10 +830,12 @@ header_params = 0; ignored = NULL; + u = r->upstream; + flcf = ngx_http_get_module_loc_conf(r, ngx_http_fastcgi_module); #if (NGX_HTTP_CACHE) - params = r->upstream->cacheable ? &flcf->params_cache : &flcf->params; + params = u->cacheable ? &flcf->params_cache : &flcf->params; #else params = &flcf->params; #endif @@ -1134,12 +1156,17 @@ h->padding_length = 0; h->reserved = 0; - h = (ngx_http_fastcgi_header_t *) b->last; - b->last += sizeof(ngx_http_fastcgi_header_t); + if (r->request_body_no_buffering) { + + u->request_bufs = cl; + + u->output.output_filter = ngx_http_fastcgi_body_output_filter; + u->output.filter_ctx = r; - if (flcf->upstream.pass_request_body) { - body = r->upstream->request_bufs; - r->upstream->request_bufs = cl; + } else if (flcf->upstream.pass_request_body) { + + body = u->request_bufs; + u->request_bufs = cl; #if (NGX_SUPPRESS_WARN) file_pos = 0; @@ -1194,6 +1221,9 @@ padding = 8 - len % 8; padding = (padding == 8) ? 0 : padding; + h = (ngx_http_fastcgi_header_t *) cl->buf->last; + cl->buf->last += sizeof(ngx_http_fastcgi_header_t); + h->version = 1; h->type = NGX_HTTP_FASTCGI_STDIN; h->request_id_hi = 0; @@ -1223,9 +1253,6 @@ b->last += padding; } - h = (ngx_http_fastcgi_header_t *) b->last; - b->last += sizeof(ngx_http_fastcgi_header_t); - cl->next = ngx_alloc_chain_link(r->pool); if (cl->next == NULL) { return NGX_ERROR; @@ -1240,17 +1267,22 @@ } } else { - r->upstream->request_bufs = cl; + u->request_bufs = cl; } - h->version = 1; - h->type = NGX_HTTP_FASTCGI_STDIN; - h->request_id_hi = 0; - h->request_id_lo = 1; - h->content_length_hi = 0; - h->content_length_lo = 0; - h->padding_length = 0; - h->reserved = 0; + if (!r->request_body_no_buffering) { + h = (ngx_http_fastcgi_header_t *) cl->buf->last; + cl->buf->last += sizeof(ngx_http_fastcgi_header_t); + + h->version = 1; + h->type = NGX_HTTP_FASTCGI_STDIN; + h->request_id_hi = 0; + h->request_id_lo = 1; + h->content_length_hi = 0; + h->content_length_lo = 0; + h->padding_length = 0; + h->reserved = 0; + } cl->next = NULL; @@ -1284,6 +1316,294 @@ static ngx_int_t +ngx_http_fastcgi_body_output_filter(void *data, ngx_chain_t *in) +{ + ngx_http_request_t *r = data; + + off_t file_pos; + u_char *pos, *start; + size_t len, padding; + ngx_buf_t *b; + ngx_int_t rc; + ngx_uint_t next, last; + ngx_chain_t *cl, *tl, *out, **ll; + ngx_http_fastcgi_ctx_t *f; + ngx_http_fastcgi_header_t *h; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "fastcgi output filter"); + + f = ngx_http_get_module_ctx(r, ngx_http_fastcgi_module); + + if (in == NULL) { + out = in; + goto out; + } + + out = NULL; + ll = &out; + + if (!f->header_sent) { + /* first buffer contains headers, pass it unmodified */ + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "fastcgi output header"); + + f->header_sent = 1; + + tl = ngx_alloc_chain_link(r->pool); + if (tl == NULL) { + return NGX_ERROR; + } + + tl->buf = in->buf; + *ll = tl; + ll = &tl->next; + + in = in->next; + + if (in == NULL) { + tl->next = NULL; + goto out; + } + } + + cl = ngx_chain_get_free_buf(r->pool, &f->free); + if (cl == NULL) { + return NGX_ERROR; + } + + b = cl->buf; + + b->tag = (ngx_buf_tag_t) &ngx_http_fastcgi_body_output_filter; + b->temporary = 1; + + if (b->start == NULL) { + /* reserve space for maximum possible padding, 7 bytes */ + + b->start = ngx_palloc(r->pool, + sizeof(ngx_http_fastcgi_header_t) + 7); + if (b->start == NULL) { + return NGX_ERROR; + } + + b->pos = b->start; + b->last = b->start; + + b->end = b->start + sizeof(ngx_http_fastcgi_header_t) + 7; + } + + *ll = cl; + + last = 0; + padding = 0; + +#if (NGX_SUPPRESS_WARN) + file_pos = 0; + pos = NULL; +#endif + + while (in) { + + ngx_log_debug7(NGX_LOG_DEBUG_EVENT, r->connection->log, 0, + "fastcgi output in l:%d f:%d %p, pos %p, size: %z " + "file: %O, size: %O", + in->buf->last_buf, + in->buf->in_file, + in->buf->start, in->buf->pos, + in->buf->last - in->buf->pos, + in->buf->file_pos, + in->buf->file_last - in->buf->file_pos); + + if (in->buf->last_buf) { + last = 1; + } + + if (ngx_buf_special(in->buf)) { + in = in->next; + continue; + } + + if (in->buf->in_file) { + file_pos = in->buf->file_pos; + + } else { + pos = in->buf->pos; + } + + next = 0; + + do { + tl = ngx_chain_get_free_buf(r->pool, &f->free); + if (tl == NULL) { + return NGX_ERROR; + } + + b = tl->buf; + start = b->start; + + ngx_memcpy(b, in->buf, sizeof(ngx_buf_t)); + + /* + * restore b->start to preserve memory allocated in the buffer, + * to reuse it later for headers and padding + */ + + b->start = start; + + if (in->buf->in_file) { + b->file_pos = file_pos; + file_pos += 32 * 1024; + + if (file_pos >= in->buf->file_last) { + file_pos = in->buf->file_last; + next = 1; + } + + b->file_last = file_pos; + len = (ngx_uint_t) (file_pos - b->file_pos); + + } else { + b->pos = pos; + pos += 32 * 1024; + + if (pos >= in->buf->last) { + pos = in->buf->last; + next = 1; + } + + b->last = pos; + len = (ngx_uint_t) (pos - b->pos); + } + + b->tag = (ngx_buf_tag_t) &ngx_http_fastcgi_body_output_filter; + b->shadow = in->buf; + b->last_shadow = next; + + b->last_buf = 0; + b->last_in_chain = 0; + + padding = 8 - len % 8; + padding = (padding == 8) ? 0 : padding; + + h = (ngx_http_fastcgi_header_t *) cl->buf->last; + cl->buf->last += sizeof(ngx_http_fastcgi_header_t); + + h->version = 1; + h->type = NGX_HTTP_FASTCGI_STDIN; + h->request_id_hi = 0; + h->request_id_lo = 1; + h->content_length_hi = (u_char) ((len >> 8) & 0xff); + h->content_length_lo = (u_char) (len & 0xff); + h->padding_length = (u_char) padding; + h->reserved = 0; + + cl->next = tl; + cl = tl; + + tl = ngx_chain_get_free_buf(r->pool, &f->free); + if (tl == NULL) { + return NGX_ERROR; + } + + b = tl->buf; + + b->tag = (ngx_buf_tag_t) &ngx_http_fastcgi_body_output_filter; + b->temporary = 1; + + if (b->start == NULL) { + /* reserve space for maximum possible padding, 7 bytes */ + + b->start = ngx_palloc(r->pool, + sizeof(ngx_http_fastcgi_header_t) + 7); + if (b->start == NULL) { + return NGX_ERROR; + } + + b->pos = b->start; + b->last = b->start; + + b->end = b->start + sizeof(ngx_http_fastcgi_header_t) + 7; + } + + if (padding) { + ngx_memzero(b->last, padding); + b->last += padding; + } + + cl->next = tl; + cl = tl; + + } while (!next); + + in = in->next; + } + + if (last) { + h = (ngx_http_fastcgi_header_t *) cl->buf->last; + cl->buf->last += sizeof(ngx_http_fastcgi_header_t); + + h->version = 1; + h->type = NGX_HTTP_FASTCGI_STDIN; + h->request_id_hi = 0; + h->request_id_lo = 1; + h->content_length_hi = 0; + h->content_length_lo = 0; + h->padding_length = 0; + h->reserved = 0; + + cl->buf->last_buf = 1; + + } else if (padding == 0) { + /* TODO: do not allocate buffers instead */ + cl->buf->temporary = 0; + cl->buf->sync = 1; + } + + cl->next = NULL; + +out: + +#if (NGX_DEBUG) + + for (cl = out; cl; cl = cl->next) { + ngx_log_debug7(NGX_LOG_DEBUG_EVENT, r->connection->log, 0, + "fastcgi output out l:%d f:%d %p, pos %p, size: %z " + "file: %O, size: %O", + cl->buf->last_buf, + cl->buf->in_file, + cl->buf->start, cl->buf->pos, + cl->buf->last - cl->buf->pos, + cl->buf->file_pos, + cl->buf->file_last - cl->buf->file_pos); + } + +#endif + + rc = ngx_chain_writer(&r->upstream->writer, out); + + ngx_chain_update_chains(r->pool, &f->free, &f->busy, &out, + (ngx_buf_tag_t) &ngx_http_fastcgi_body_output_filter); + + for (cl = f->free; cl; cl = cl->next) { + + /* mark original buffers as sent */ + + if (cl->buf->shadow) { + if (cl->buf->last_shadow) { + b = cl->buf->shadow; + b->pos = b->last; + } + + cl->buf->shadow = NULL; + } + } + + return rc; +} + + +static ngx_int_t ngx_http_fastcgi_process_header(ngx_http_request_t *r) { u_char *p, *msg, *start, *last, @@ -2405,6 +2725,7 @@ conf->upstream.store_access = NGX_CONF_UNSET_UINT; conf->upstream.next_upstream_tries = NGX_CONF_UNSET_UINT; conf->upstream.buffering = NGX_CONF_UNSET; + conf->upstream.request_buffering = NGX_CONF_UNSET; conf->upstream.ignore_client_abort = NGX_CONF_UNSET; conf->upstream.force_ranges = NGX_CONF_UNSET; @@ -2498,6 +2819,9 @@ ngx_conf_merge_value(conf->upstream.buffering, prev->upstream.buffering, 1); + ngx_conf_merge_value(conf->upstream.request_buffering, + prev->upstream.request_buffering, 1); + ngx_conf_merge_value(conf->upstream.ignore_client_abort, prev->upstream.ignore_client_abort, 0);

@@ -110,7 +110,12 @@ ngx_http_proxy_vars_t vars; off_t internal_body_length; - ngx_uint_t head; /* unsigned head:1 */ + ngx_chain_t *free; + ngx_chain_t *busy; + + unsigned head:1; + unsigned internal_chunked:1; + unsigned header_sent:1; } ngx_http_proxy_ctx_t; @@ -121,6 +126,7 @@ #endif static ngx_int_t ngx_http_proxy_create_request(ngx_http_request_t *r); static ngx_int_t ngx_http_proxy_reinit_request(ngx_http_request_t *r); +static ngx_int_t ngx_http_proxy_body_output_filter(void *data, ngx_chain_t *in); static ngx_int_t ngx_http_proxy_process_status_line(ngx_http_request_t *r); static ngx_int_t ngx_http_proxy_process_header(ngx_http_request_t *r); static ngx_int_t ngx_http_proxy_input_filter_init(void *data); @@ -146,6 +152,8 @@ static ngx_int_t ngx_http_proxy_internal_body_length_variable(ngx_http_request_t *r, ngx_http_variable_value_t *v, uintptr_t data); +static ngx_int_t ngx_http_proxy_internal_chunked_variable(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data); static ngx_int_t ngx_http_proxy_rewrite_redirect(ngx_http_request_t *r, ngx_table_elt_t *h, size_t prefix); static ngx_int_t ngx_http_proxy_rewrite_cookie(ngx_http_request_t *r, @@ -292,6 +300,13 @@ offsetof(ngx_http_proxy_loc_conf_t, upstream.buffering), NULL }, + { ngx_string("proxy_request_buffering"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_proxy_loc_conf_t, upstream.request_buffering), + NULL }, + { ngx_string("proxy_ignore_client_abort"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, ngx_conf_set_flag_slot, @@ -721,8 +736,8 @@ { ngx_string("Host"), ngx_string("$proxy_host") }, { ngx_string("Connection"), ngx_string("close") }, { ngx_string("Content-Length"), ngx_string("$proxy_internal_body_length") }, + { ngx_string("Transfer-Encoding"), ngx_string("$proxy_internal_chunked") }, { ngx_string("TE"), ngx_string("") }, - { ngx_string("Transfer-Encoding"), ngx_string("") }, { ngx_string("Keep-Alive"), ngx_string("") }, { ngx_string("Expect"), ngx_string("") }, { ngx_string("Upgrade"), ngx_string("") }, @@ -749,8 +764,8 @@ { ngx_string("Host"), ngx_string("$proxy_host") }, { ngx_string("Connection"), ngx_string("close") }, { ngx_string("Content-Length"), ngx_string("$proxy_internal_body_length") }, + { ngx_string("Transfer-Encoding"), ngx_string("$proxy_internal_chunked") }, { ngx_string("TE"), ngx_string("") }, - { ngx_string("Transfer-Encoding"), ngx_string("") }, { ngx_string("Keep-Alive"), ngx_string("") }, { ngx_string("Expect"), ngx_string("") }, { ngx_string("Upgrade"), ngx_string("") }, @@ -786,6 +801,10 @@ ngx_http_proxy_internal_body_length_variable, 0, NGX_HTTP_VAR_NOCACHEABLE|NGX_HTTP_VAR_NOHASH, 0 }, + { ngx_string("proxy_internal_chunked"), NULL, + ngx_http_proxy_internal_chunked_variable, 0, + NGX_HTTP_VAR_NOCACHEABLE|NGX_HTTP_VAR_NOHASH, 0 }, + { ngx_null_string, NULL, NULL, 0, 0, 0 } }; @@ -812,7 +831,7 @@ ctx = ngx_pcalloc(r->pool, sizeof(ngx_http_proxy_ctx_t)); if (ctx == NULL) { - return NGX_ERROR; + return NGX_HTTP_INTERNAL_SERVER_ERROR; } ngx_http_set_ctx(r, ctx, ngx_http_proxy_module); @@ -876,6 +895,14 @@ u->accel = 1; + if (!plcf->upstream.request_buffering + && plcf->body_values == NULL && plcf->upstream.pass_request_body + && (!r->headers_in.chunked + || plcf->http_version == NGX_HTTP_VERSION_11)) + { + r->request_body_no_buffering = 1; + } + rc = ngx_http_read_client_request_body(r, ngx_http_upstream_init); if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { @@ -1194,6 +1221,10 @@ ctx->internal_body_length = body_len; len += body_len; + } else if (r->headers_in.chunked && r->reading_body) { + ctx->internal_body_length = -1; + ctx->internal_chunked = 1; + } else { ctx->internal_body_length = r->headers_in.content_length_n; } @@ -1379,6 +1410,7 @@ if (plcf->body_values) { e.ip = plcf->body_values->elts; e.pos = b->last; + e.skip = 0; while (*(uintptr_t *) e.ip) { code = *(ngx_http_script_code_pt *) e.ip; @@ -1392,7 +1424,16 @@ "http proxy header:%N\"%*s\"", (size_t) (b->last - b->pos), b->pos); - if (plcf->body_values == NULL && plcf->upstream.pass_request_body) { + if (r->request_body_no_buffering) { + + u->request_bufs = cl; + + if (ctx->internal_chunked) { + u->output.output_filter = ngx_http_proxy_body_output_filter; + u->output.filter_ctx = r; + } + + } else if (plcf->body_values == NULL && plcf->upstream.pass_request_body) { body = u->request_bufs; u->request_bufs = cl; @@ -1454,6 +1495,172 @@ static ngx_int_t +ngx_http_proxy_body_output_filter(void *data, ngx_chain_t *in) +{ + ngx_http_request_t *r = data; + + off_t size; + u_char *chunk; + ngx_int_t rc; + ngx_buf_t *b; + ngx_chain_t *out, *cl, *tl, **ll; + ngx_http_proxy_ctx_t *ctx; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "proxy output filter"); + + ctx = ngx_http_get_module_ctx(r, ngx_http_proxy_module); + + if (in == NULL) { + out = in; + goto out; + } + + out = NULL; + ll = &out; + + if (!ctx->header_sent) { + /* first buffer contains headers, pass it unmodified */ + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "proxy output header"); + + ctx->header_sent = 1; + + tl = ngx_alloc_chain_link(r->pool); + if (tl == NULL) { + return NGX_ERROR; + } + + tl->buf = in->buf; + *ll = tl; + ll = &tl->next; + + in = in->next; + + if (in == NULL) { + tl->next = NULL; + goto out; + } + } + + size = 0; + cl = in; + + for ( ;; ) { + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "proxy output chunk: %d", ngx_buf_size(cl->buf)); + + size += ngx_buf_size(cl->buf); + + if (cl->buf->flush + || cl->buf->sync + || ngx_buf_in_memory(cl->buf) + || cl->buf->in_file) + { + tl = ngx_alloc_chain_link(r->pool); + if (tl == NULL) { + return NGX_ERROR; + } + + tl->buf = cl->buf; + *ll = tl; + ll = &tl->next; + } + + if (cl->next == NULL) { + break; + } + + cl = cl->next; + } + + if (size) { + tl = ngx_chain_get_free_buf(r->pool, &ctx->free); + if (tl == NULL) { + return NGX_ERROR; + } + + b = tl->buf; + chunk = b->start; + + if (chunk == NULL) { + /* the "0000000000000000" is 64-bit hexadecimal string */ + + chunk = ngx_palloc(r->pool, sizeof("0000000000000000" CRLF) - 1); + if (chunk == NULL) { + return NGX_ERROR; + } + + b->start = chunk; + b->end = chunk + sizeof("0000000000000000" CRLF) - 1; + } + + b->tag = (ngx_buf_tag_t) &ngx_http_proxy_body_output_filter; + b->memory = 0; + b->temporary = 1; + b->pos = chunk; + b->last = ngx_sprintf(chunk, "%xO" CRLF, size); + + tl->next = out; + out = tl; + } + + if (cl->buf->last_buf) { + tl = ngx_chain_get_free_buf(r->pool, &ctx->free); + if (tl == NULL) { + return NGX_ERROR; + } + + b = tl->buf; + + b->tag = (ngx_buf_tag_t) &ngx_http_proxy_body_output_filter; + b->temporary = 0; + b->memory = 1; + b->last_buf = 1; + b->pos = (u_char *) CRLF "0" CRLF CRLF; + b->last = b->pos + 7; + + cl->buf->last_buf = 0; + + *ll = tl; + + if (size == 0) { + b->pos += 2; + } + + } else if (size > 0) { + tl = ngx_chain_get_free_buf(r->pool, &ctx->free); + if (tl == NULL) { + return NGX_ERROR; + } + + b = tl->buf; + + b->tag = (ngx_buf_tag_t) &ngx_http_proxy_body_output_filter; + b->temporary = 0; + b->memory = 1; + b->pos = (u_char *) CRLF; + b->last = b->pos + 2; + + *ll = tl; + + } else { + *ll = NULL; + } + +out: + + rc = ngx_chain_writer(&r->upstream->writer, out); + + ngx_chain_update_chains(r->pool, &ctx->free, &ctx->busy, &out, + (ngx_buf_tag_t) &ngx_http_proxy_body_output_filter); + + return rc; +} + + +static ngx_int_t ngx_http_proxy_process_status_line(ngx_http_request_t *r) { size_t len; @@ -2247,6 +2454,30 @@ static ngx_int_t +ngx_http_proxy_internal_chunked_variable(ngx_http_request_t *r, + ngx_http_variable_value_t *v, uintptr_t data) +{ + ngx_http_proxy_ctx_t *ctx; + + ctx = ngx_http_get_module_ctx(r, ngx_http_proxy_module); + + if (ctx == NULL || !ctx->internal_chunked) { + v->not_found = 1; + return NGX_OK; + } + + v->valid = 1; + v->no_cacheable = 0; + v->not_found = 0; + + v->data = (u_char *) "chunked"; + v->len = sizeof("chunked") - 1; + + return NGX_OK; +} + + +static ngx_int_t ngx_http_proxy_rewrite_redirect(ngx_http_request_t *r, ngx_table_elt_t *h, size_t prefix) { @@ -2581,6 +2812,7 @@ conf->upstream.store_access = NGX_CONF_UNSET_UINT; conf->upstream.next_upstream_tries = NGX_CONF_UNSET_UINT; conf->upstream.buffering = NGX_CONF_UNSET; + conf->upstream.request_buffering = NGX_CONF_UNSET; conf->upstream.ignore_client_abort = NGX_CONF_UNSET; conf->upstream.force_ranges = NGX_CONF_UNSET; @@ -2690,6 +2922,9 @@ ngx_conf_merge_value(conf->upstream.buffering, prev->upstream.buffering, 1); + ngx_conf_merge_value(conf->upstream.request_buffering, + prev->upstream.request_buffering, 1); + ngx_conf_merge_value(conf->upstream.ignore_client_abort, prev->upstream.ignore_client_abort, 0);

@@ -274,7 +274,7 @@ ngx_uint_t ranges) { u_char *p; - off_t start, end, size, content_length; + off_t start, end, size, content_length, cutoff, cutlim; ngx_uint_t suffix; ngx_http_range_t *range; @@ -282,6 +282,9 @@ size = 0; content_length = r->headers_out.content_length_n; + cutoff = NGX_MAX_OFF_T_VALUE / 10; + cutlim = NGX_MAX_OFF_T_VALUE % 10; + for ( ;; ) { start = 0; end = 0; @@ -295,6 +298,10 @@ } while (*p >= '0' && *p <= '9') { + if (start >= cutoff && (start > cutoff || *p - '0' > cutlim)) { + return NGX_HTTP_RANGE_NOT_SATISFIABLE; + } + start = start * 10 + *p++ - '0'; } @@ -321,6 +328,10 @@ } while (*p >= '0' && *p <= '9') { + if (end >= cutoff && (end > cutoff || *p - '0' > cutlim)) { + return NGX_HTTP_RANGE_NOT_SATISFIABLE; + } + end = end * 10 + *p++ - '0'; }

@@ -120,6 +120,13 @@ offsetof(ngx_http_scgi_loc_conf_t, upstream.buffering), NULL }, + { ngx_string("scgi_request_buffering"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_scgi_loc_conf_t, upstream.request_buffering), + NULL }, + { ngx_string("scgi_ignore_client_abort"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, ngx_conf_set_flag_slot, @@ -504,6 +511,13 @@ u->pipe->input_filter = ngx_event_pipe_copy_input_filter; u->pipe->input_ctx = r; + if (!scf->upstream.request_buffering + && scf->upstream.pass_request_body + && !r->headers_in.chunked) + { + r->request_body_no_buffering = 1; + } + rc = ngx_http_read_client_request_body(r, ngx_http_upstream_init); if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { @@ -865,7 +879,10 @@ *b->last++ = (u_char) ','; - if (scf->upstream.pass_request_body) { + if (r->request_body_no_buffering) { + r->upstream->request_bufs = cl; + + } else if (scf->upstream.pass_request_body) { body = r->upstream->request_bufs; r->upstream->request_bufs = cl; @@ -1162,6 +1179,7 @@ conf->upstream.store_access = NGX_CONF_UNSET_UINT; conf->upstream.next_upstream_tries = NGX_CONF_UNSET_UINT; conf->upstream.buffering = NGX_CONF_UNSET; + conf->upstream.request_buffering = NGX_CONF_UNSET; conf->upstream.ignore_client_abort = NGX_CONF_UNSET; conf->upstream.force_ranges = NGX_CONF_UNSET; @@ -1250,6 +1268,9 @@ ngx_conf_merge_value(conf->upstream.buffering, prev->upstream.buffering, 1); + ngx_conf_merge_value(conf->upstream.request_buffering, + prev->upstream.request_buffering, 1); + ngx_conf_merge_value(conf->upstream.ignore_client_abort, prev->upstream.ignore_client_abort, 0);

@@ -715,8 +715,10 @@ SSL_CTX_set_options(conf->ssl.ctx, SSL_OP_CIPHER_SERVER_PREFERENCE); } +#ifndef LIBRESSL_VERSION_NUMBER /* a temporary 512-bit RSA key is required for export versions of MSIE */ SSL_CTX_set_tmp_rsa_callback(conf->ssl.ctx, ngx_ssl_rsa512_key_callback); +#endif if (ngx_ssl_dhparam(cf, &conf->ssl, &conf->dhparam) != NGX_OK) { return NGX_CONF_ERROR;

@@ -49,8 +49,8 @@ static ngx_int_t ngx_http_upstream_init_chash(ngx_conf_t *cf, ngx_http_upstream_srv_conf_t *us); -static void ngx_http_upstream_add_chash_point( - ngx_http_upstream_chash_points_t *points, uint32_t hash, ngx_str_t *server); +static int ngx_libc_cdecl + ngx_http_upstream_chash_cmp_points(const void *one, const void *two); static ngx_uint_t ngx_http_upstream_find_chash_point( ngx_http_upstream_chash_points_t *points, uint32_t hash); static ngx_int_t ngx_http_upstream_init_chash_peer(ngx_http_request_t *r, @@ -360,12 +360,27 @@ ngx_crc32_update(&hash, (u_char *) &prev_hash, sizeof(uint32_t)); ngx_crc32_final(hash); - ngx_http_upstream_add_chash_point(points, hash, &peer->server); + points->point[points->number].hash = hash; + points->point[points->number].server = server; + points->number++; prev_hash = hash; } } + ngx_qsort(points->point, + points->number, + sizeof(ngx_http_upstream_chash_point_t), + ngx_http_upstream_chash_cmp_points); + + for (i = 0, j = 1; j < points->number; j++) { + if (points->point[i].hash != points->point[j].hash) { + points->point[++i] = points->point[j]; + } + } + + points->number = i + 1; + hcf = ngx_http_conf_upstream_srv_conf(us, ngx_http_upstream_hash_module); hcf->points = points; @@ -373,28 +388,23 @@ } -static void -ngx_http_upstream_add_chash_point(ngx_http_upstream_chash_points_t *points, - uint32_t hash, ngx_str_t *server) +static int ngx_libc_cdecl +ngx_http_upstream_chash_cmp_points(const void *one, const void *two) { - size_t size; - ngx_uint_t i; - ngx_http_upstream_chash_point_t *point; + ngx_http_upstream_chash_point_t *first = + (ngx_http_upstream_chash_point_t *) one; + ngx_http_upstream_chash_point_t *second = + (ngx_http_upstream_chash_point_t *) two; - i = ngx_http_upstream_find_chash_point(points, hash); - point = &points->point[i]; + if (first->hash < second->hash) { + return -1; - if (point->hash == hash) { - return; - } - - size = (points->number - i) * sizeof(ngx_http_upstream_chash_point_t); + } else if (first->hash > second->hash) { + return 1; - ngx_memmove(point + 1, point, size); - - points->number++; - point->hash = hash; - point->server = server; + } else { + return 0; + } }

@@ -387,7 +387,7 @@ n = recv(c->fd, buf, 1, MSG_PEEK); if (n == -1 && ngx_socket_errno == NGX_EAGAIN) { - /* stale event */ + ev->ready = 0; if (ngx_handle_read_event(c->read, 0) != NGX_OK) { goto close;

@@ -180,6 +180,13 @@ offsetof(ngx_http_uwsgi_loc_conf_t, upstream.buffering), NULL }, + { ngx_string("uwsgi_request_buffering"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_uwsgi_loc_conf_t, upstream.request_buffering), + NULL }, + { ngx_string("uwsgi_ignore_client_abort"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, ngx_conf_set_flag_slot, @@ -672,6 +679,13 @@ u->pipe->input_filter = ngx_event_pipe_copy_input_filter; u->pipe->input_ctx = r; + if (!uwcf->upstream.request_buffering + && uwcf->upstream.pass_request_body + && !r->headers_in.chunked) + { + r->request_body_no_buffering = 1; + } + rc = ngx_http_read_client_request_body(r, ngx_http_upstream_init); if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { @@ -1068,7 +1082,10 @@ b->last = ngx_copy(b->last, uwcf->uwsgi_string.data, uwcf->uwsgi_string.len); - if (uwcf->upstream.pass_request_body) { + if (r->request_body_no_buffering) { + r->upstream->request_bufs = cl; + + } else if (uwcf->upstream.pass_request_body) { body = r->upstream->request_bufs; r->upstream->request_bufs = cl; @@ -1368,6 +1385,7 @@ conf->upstream.store_access = NGX_CONF_UNSET_UINT; conf->upstream.next_upstream_tries = NGX_CONF_UNSET_UINT; conf->upstream.buffering = NGX_CONF_UNSET; + conf->upstream.request_buffering = NGX_CONF_UNSET; conf->upstream.ignore_client_abort = NGX_CONF_UNSET; conf->upstream.force_ranges = NGX_CONF_UNSET; @@ -1464,6 +1482,9 @@ ngx_conf_merge_value(conf->upstream.buffering, prev->upstream.buffering, 1); + ngx_conf_merge_value(conf->upstream.request_buffering, + prev->upstream.request_buffering, 1); + ngx_conf_merge_value(conf->upstream.ignore_client_abort, prev->upstream.ignore_client_abort, 0);

@@ -69,8 +69,9 @@ ngx_uint_t ngx_http_max_module; -ngx_int_t (*ngx_http_top_header_filter) (ngx_http_request_t *r); -ngx_int_t (*ngx_http_top_body_filter) (ngx_http_request_t *r, ngx_chain_t *ch); +ngx_http_output_header_filter_pt ngx_http_top_header_filter; +ngx_http_output_body_filter_pt ngx_http_top_body_filter; +ngx_http_request_body_filter_pt ngx_http_top_request_body_filter; ngx_str_t ngx_http_html_default_types[] = {

@@ -36,7 +36,6 @@ #include <ngx_http_script.h> #include <ngx_http_upstream.h> #include <ngx_http_upstream_round_robin.h> -#include <ngx_http_busy_lock.h> #include <ngx_http_core_module.h> #if (NGX_HTTP_SPDY) @@ -131,9 +130,6 @@ void ngx_http_request_empty_handler(ngx_http_request_t *r); -#define ngx_http_ephemeral(r) (void *) (&r->uri_start) - - #define NGX_HTTP_LAST 1 #define NGX_HTTP_FLUSH 2 @@ -142,6 +138,7 @@ ngx_int_t ngx_http_read_client_request_body(ngx_http_request_t *r, ngx_http_client_body_handler_pt post_handler); +ngx_int_t ngx_http_read_unbuffered_request_body(ngx_http_request_t *r); ngx_int_t ngx_http_send_header(ngx_http_request_t *r); ngx_int_t ngx_http_special_response_handler(ngx_http_request_t *r, @@ -181,6 +178,7 @@ extern ngx_http_output_header_filter_pt ngx_http_top_header_filter; extern ngx_http_output_body_filter_pt ngx_http_top_body_filter; +extern ngx_http_request_body_filter_pt ngx_http_top_request_body_filter; #endif /* _NGX_HTTP_H_INCLUDED_ */

@@ -142,6 +142,7 @@ ngx_slab_pool_t *shpool; ngx_path_t *path; + ngx_path_t *temp_path; off_t max_size; size_t bsize; @@ -162,7 +163,7 @@ ngx_int_t ngx_http_file_cache_create(ngx_http_request_t *r); void ngx_http_file_cache_create_key(ngx_http_request_t *r); ngx_int_t ngx_http_file_cache_open(ngx_http_request_t *r); -void ngx_http_file_cache_set_header(ngx_http_request_t *r, u_char *buf); +ngx_int_t ngx_http_file_cache_set_header(ngx_http_request_t *r, u_char *buf); void ngx_http_file_cache_update(ngx_http_request_t *r, ngx_temp_file_t *tf); void ngx_http_file_cache_update_header(ngx_http_request_t *r); ngx_int_t ngx_http_cache_send(ngx_http_request_t *);

@@ -20,9 +20,15 @@ ngx_file_t *file); static void ngx_http_copy_aio_event_handler(ngx_event_t *ev); #if (NGX_HAVE_AIO_SENDFILE) +static ssize_t ngx_http_copy_aio_sendfile_preload(ngx_buf_t *file); static void ngx_http_copy_aio_sendfile_event_handler(ngx_event_t *ev); #endif #endif +#if (NGX_THREADS) +static ngx_int_t ngx_http_copy_thread_handler(ngx_thread_task_t *task, + ngx_file_t *file); +static void ngx_http_copy_thread_event_handler(ngx_event_t *ev); +#endif static void *ngx_http_copy_filter_create_conf(ngx_conf_t *cf); static char *ngx_http_copy_filter_merge_conf(ngx_conf_t *cf, @@ -120,91 +126,42 @@ ctx->filter_ctx = r; #if (NGX_HAVE_FILE_AIO) - if (ngx_file_aio) { - if (clcf->aio) { - ctx->aio_handler = ngx_http_copy_aio_handler; - } + if (ngx_file_aio && clcf->aio == NGX_HTTP_AIO_ON) { + ctx->aio_handler = ngx_http_copy_aio_handler; #if (NGX_HAVE_AIO_SENDFILE) - c->aio_sendfile = (clcf->aio == NGX_HTTP_AIO_SENDFILE); + ctx->aio_preload = ngx_http_copy_aio_sendfile_preload; #endif } #endif +#if (NGX_THREADS) + if (clcf->aio == NGX_HTTP_AIO_THREADS) { + ctx->thread_handler = ngx_http_copy_thread_handler; + } +#endif + if (in && in->buf && ngx_buf_size(in->buf)) { r->request_output = 1; } } -#if (NGX_HAVE_FILE_AIO) +#if (NGX_HAVE_FILE_AIO || NGX_THREADS) ctx->aio = r->aio; #endif - for ( ;; ) { - rc = ngx_output_chain(ctx, in); - - if (ctx->in == NULL) { - r->buffered &= ~NGX_HTTP_COPY_BUFFERED; - - } else { - r->buffered |= NGX_HTTP_COPY_BUFFERED; - } - - ngx_log_debug3(NGX_LOG_DEBUG_HTTP, c->log, 0, - "http copy filter: %i \"%V?%V\"", rc, &r->uri, &r->args); - -#if (NGX_HAVE_FILE_AIO && NGX_HAVE_AIO_SENDFILE) - - if (c->busy_sendfile) { - ssize_t n; - off_t offset; - ngx_file_t *file; - ngx_http_ephemeral_t *e; - - if (r->aio) { - c->busy_sendfile = NULL; - return rc; - } - - file = c->busy_sendfile->file; - offset = c->busy_sendfile->file_pos; - - if (file->aio) { - c->busy_count = (offset == file->aio->last_offset) ? - c->busy_count + 1 : 0; - file->aio->last_offset = offset; - - if (c->busy_count > 2) { - ngx_log_error(NGX_LOG_ALERT, c->log, 0, - "sendfile(%V) returned busy again", - &file->name); - c->aio_sendfile = 0; - } - } - - c->busy_sendfile = NULL; - e = (ngx_http_ephemeral_t *) &r->uri_start; + rc = ngx_output_chain(ctx, in); - n = ngx_file_aio_read(file, &e->aio_preload, 1, offset, r->pool); + if (ctx->in == NULL) { + r->buffered &= ~NGX_HTTP_COPY_BUFFERED; - if (n > 0) { - in = NULL; - continue; - } - - rc = n; - - if (rc == NGX_AGAIN) { - file->aio->data = r; - file->aio->handler = ngx_http_copy_aio_sendfile_event_handler; + } else { + r->buffered |= NGX_HTTP_COPY_BUFFERED; + } - r->main->blocked++; - r->aio = 1; - } - } -#endif + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, c->log, 0, + "http copy filter: %i \"%V?%V\"", rc, &r->uri, &r->args); - return rc; - } + return rc; } @@ -244,6 +201,29 @@ #if (NGX_HAVE_AIO_SENDFILE) +static ssize_t +ngx_http_copy_aio_sendfile_preload(ngx_buf_t *file) +{ + ssize_t n; + static u_char buf[1]; + ngx_event_aio_t *aio; + ngx_http_request_t *r; + + n = ngx_file_aio_read(file->file, buf, 1, file->file_pos, NULL); + + if (n == NGX_AGAIN) { + aio = file->file->aio; + aio->handler = ngx_http_copy_aio_sendfile_event_handler; + + r = aio->data; + r->main->blocked++; + r->aio = 1; + } + + return n; +} + + static void ngx_http_copy_aio_sendfile_event_handler(ngx_event_t *ev) { @@ -264,6 +244,67 @@ #endif +#if (NGX_THREADS) + +static ngx_int_t +ngx_http_copy_thread_handler(ngx_thread_task_t *task, ngx_file_t *file) +{ + ngx_str_t name; + ngx_thread_pool_t *tp; + ngx_http_request_t *r; + ngx_http_core_loc_conf_t *clcf; + + r = file->thread_ctx; + + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + tp = clcf->thread_pool; + + if (tp == NULL) { + if (ngx_http_complex_value(r, clcf->thread_pool_value, &name) + != NGX_OK) + { + return NGX_ERROR; + } + + tp = ngx_thread_pool_get((ngx_cycle_t *) ngx_cycle, &name); + + if (tp == NULL) { + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "thread pool \"%V\" not found", &name); + return NGX_ERROR; + } + } + + task->event.data = r; + task->event.handler = ngx_http_copy_thread_event_handler; + + if (ngx_thread_task_post(tp, task) != NGX_OK) { + return NGX_ERROR; + } + + r->main->blocked++; + r->aio = 1; + + return NGX_OK; +} + + +static void +ngx_http_copy_thread_event_handler(ngx_event_t *ev) +{ + ngx_http_request_t *r; + + r = ev->data; + + r->main->blocked--; + r->aio = 0; + + r->connection->write->handler(r->connection->write); +} + +#endif + + static void * ngx_http_copy_filter_create_conf(ngx_conf_t *cf) {

@@ -26,6 +26,7 @@ ngx_http_location_tree_node_t *node); static ngx_int_t ngx_http_core_preconfiguration(ngx_conf_t *cf); +static ngx_int_t ngx_http_core_postconfiguration(ngx_conf_t *cf); static void *ngx_http_core_create_main_conf(ngx_conf_t *cf); static char *ngx_http_core_init_main_conf(ngx_conf_t *cf, void *conf); static void *ngx_http_core_create_srv_conf(ngx_conf_t *cf); @@ -54,6 +55,8 @@ static char *ngx_http_core_root(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char *ngx_http_core_limit_except(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); +static char *ngx_http_core_set_aio(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); static char *ngx_http_core_directio(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static char *ngx_http_core_error_page(ngx_conf_t *cf, ngx_command_t *cmd, @@ -114,20 +117,6 @@ }; -#if (NGX_HAVE_FILE_AIO) - -static ngx_conf_enum_t ngx_http_core_aio[] = { - { ngx_string("off"), NGX_HTTP_AIO_OFF }, - { ngx_string("on"), NGX_HTTP_AIO_ON }, -#if (NGX_HAVE_AIO_SENDFILE) - { ngx_string("sendfile"), NGX_HTTP_AIO_SENDFILE }, -#endif - { ngx_null_string, 0 } -}; - -#endif - - static ngx_conf_enum_t ngx_http_core_satisfy[] = { { ngx_string("all"), NGX_HTTP_SATISFY_ALL }, { ngx_string("any"), NGX_HTTP_SATISFY_ANY }, @@ -423,16 +412,12 @@ offsetof(ngx_http_core_loc_conf_t, sendfile_max_chunk), NULL }, -#if (NGX_HAVE_FILE_AIO) - { ngx_string("aio"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, - ngx_conf_set_enum_slot, + ngx_http_core_set_aio, NGX_HTTP_LOC_CONF_OFFSET, - offsetof(ngx_http_core_loc_conf_t, aio), - &ngx_http_core_aio }, - -#endif + 0, + NULL }, { ngx_string("read_ahead"), NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, @@ -795,7 +780,7 @@ static ngx_http_module_t ngx_http_core_module_ctx = { ngx_http_core_preconfiguration, /* preconfiguration */ - NULL, /* postconfiguration */ + ngx_http_core_postconfiguration, /* postconfiguration */ ngx_http_core_create_main_conf, /* create main configuration */ ngx_http_core_init_main_conf, /* init main configuration */ @@ -1353,7 +1338,7 @@ continue; } - if (of.is_dir && !test_dir) { + if (of.is_dir != test_dir) { continue; } @@ -3436,6 +3421,15 @@ } +static ngx_int_t +ngx_http_core_postconfiguration(ngx_conf_t *cf) +{ + ngx_http_top_request_body_filter = ngx_http_request_body_save_filter; + + return NGX_OK; +} + + static void * ngx_http_core_create_main_conf(ngx_conf_t *cf) { @@ -3639,8 +3633,10 @@ clcf->internal = NGX_CONF_UNSET; clcf->sendfile = NGX_CONF_UNSET; clcf->sendfile_max_chunk = NGX_CONF_UNSET_SIZE; -#if (NGX_HAVE_FILE_AIO) clcf->aio = NGX_CONF_UNSET; +#if (NGX_THREADS) + clcf->thread_pool = NGX_CONF_UNSET_PTR; + clcf->thread_pool_value = NGX_CONF_UNSET_PTR; #endif clcf->read_ahead = NGX_CONF_UNSET_SIZE; clcf->directio = NGX_CONF_UNSET; @@ -3857,9 +3853,14 @@ ngx_conf_merge_value(conf->sendfile, prev->sendfile, 0); ngx_conf_merge_size_value(conf->sendfile_max_chunk, prev->sendfile_max_chunk, 0); -#if (NGX_HAVE_FILE_AIO) +#if (NGX_HAVE_FILE_AIO || NGX_THREADS) ngx_conf_merge_value(conf->aio, prev->aio, NGX_HTTP_AIO_OFF); #endif +#if (NGX_THREADS) + ngx_conf_merge_ptr_value(conf->thread_pool, prev->thread_pool, NULL); + ngx_conf_merge_ptr_value(conf->thread_pool_value, prev->thread_pool_value, + NULL); +#endif ngx_conf_merge_size_value(conf->read_ahead, prev->read_ahead, 0); ngx_conf_merge_off_value(conf->directio, prev->directio, NGX_OPEN_FILE_DIRECTIO_OFF); @@ -4653,6 +4654,116 @@ } +static char * +ngx_http_core_set_aio(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_core_loc_conf_t *clcf = conf; + + ngx_str_t *value; + + if (clcf->aio != NGX_CONF_UNSET) { + return "is duplicate"; + } + +#if (NGX_THREADS) + clcf->thread_pool = NULL; + clcf->thread_pool_value = NULL; +#endif + + value = cf->args->elts; + + if (ngx_strcmp(value[1].data, "off") == 0) { + clcf->aio = NGX_HTTP_AIO_OFF; + return NGX_CONF_OK; + } + + if (ngx_strcmp(value[1].data, "on") == 0) { +#if (NGX_HAVE_FILE_AIO) + clcf->aio = NGX_HTTP_AIO_ON; + return NGX_CONF_OK; +#else + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "\"aio on\" " + "is unsupported on this platform"); + return NGX_CONF_ERROR; +#endif + } + +#if (NGX_HAVE_AIO_SENDFILE) + + if (ngx_strcmp(value[1].data, "sendfile") == 0) { + clcf->aio = NGX_HTTP_AIO_ON; + + ngx_conf_log_error(NGX_LOG_WARN, cf, 0, + "the \"sendfile\" parameter of " + "the \"aio\" directive is deprecated"); + return NGX_CONF_OK; + } + +#endif + + if (ngx_strncmp(value[1].data, "threads", 7) == 0 + && (value[1].len == 7 || value[1].data[7] == '=')) + { +#if (NGX_THREADS) + ngx_str_t name; + ngx_thread_pool_t *tp; + ngx_http_complex_value_t cv; + ngx_http_compile_complex_value_t ccv; + + clcf->aio = NGX_HTTP_AIO_THREADS; + + if (value[1].len >= 8) { + name.len = value[1].len - 8; + name.data = value[1].data + 8; + + ngx_memzero(&ccv, sizeof(ngx_http_compile_complex_value_t)); + + ccv.cf = cf; + ccv.value = &name; + ccv.complex_value = &cv; + + if (ngx_http_compile_complex_value(&ccv) != NGX_OK) { + return NGX_CONF_ERROR; + } + + if (cv.lengths != NULL) { + clcf->thread_pool_value = ngx_palloc(cf->pool, + sizeof(ngx_http_complex_value_t)); + if (clcf->thread_pool_value == NULL) { + return NGX_CONF_ERROR; + } + + *clcf->thread_pool_value = cv; + + return NGX_CONF_OK; + } + + tp = ngx_thread_pool_add(cf, &name); + + } else { + tp = ngx_thread_pool_add(cf, NULL); + } + + if (tp == NULL) { + return NGX_CONF_ERROR; + } + + clcf->thread_pool = tp; + + return NGX_CONF_OK; +#else + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "\"aio threads\" " + "is unsupported on this platform"); + return NGX_CONF_ERROR; +#endif + } + + return "invalid value"; +} + + static char * ngx_http_core_directio(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) {

@@ -13,6 +13,10 @@ #include <ngx_core.h> #include <ngx_http.h> +#if (NGX_THREADS) +#include <ngx_thread_pool.h> +#endif + #define NGX_HTTP_GZIP_PROXIED_OFF 0x0002 #define NGX_HTTP_GZIP_PROXIED_EXPIRED 0x0004 @@ -27,7 +31,7 @@ #define NGX_HTTP_AIO_OFF 0 #define NGX_HTTP_AIO_ON 1 -#define NGX_HTTP_AIO_SENDFILE 2 +#define NGX_HTTP_AIO_THREADS 2 #define NGX_HTTP_SATISFY_ALL 0 @@ -396,9 +400,7 @@ /* client_body_in_singe_buffer */ ngx_flag_t internal; /* internal */ ngx_flag_t sendfile; /* sendfile */ -#if (NGX_HAVE_FILE_AIO) ngx_flag_t aio; /* aio */ -#endif ngx_flag_t tcp_nopush; /* tcp_nopush */ ngx_flag_t tcp_nodelay; /* tcp_nodelay */ ngx_flag_t reset_timedout_connection; /* reset_timedout_connection */ @@ -424,6 +426,11 @@ #endif #endif +#if (NGX_THREADS) + ngx_thread_pool_t *thread_pool; + ngx_http_complex_value_t *thread_pool_value; +#endif + #if (NGX_HAVE_OPENAT) ngx_uint_t disable_symlinks; /* disable_symlinks */ ngx_http_complex_value_t *disable_symlinks_from; @@ -526,10 +533,14 @@ typedef ngx_int_t (*ngx_http_output_header_filter_pt)(ngx_http_request_t *r); typedef ngx_int_t (*ngx_http_output_body_filter_pt) (ngx_http_request_t *r, ngx_chain_t *chain); +typedef ngx_int_t (*ngx_http_request_body_filter_pt) + (ngx_http_request_t *r, ngx_chain_t *chain); ngx_int_t ngx_http_output_filter(ngx_http_request_t *r, ngx_chain_t *chain); ngx_int_t ngx_http_write_filter(ngx_http_request_t *r, ngx_chain_t *chain); +ngx_int_t ngx_http_request_body_save_filter(ngx_http_request_t *r, + ngx_chain_t *chain); ngx_int_t ngx_http_set_disable_symlinks(ngx_http_request_t *r,

@@ -37,6 +37,8 @@ ngx_md5_t *md5, ngx_str_t *name); static ngx_int_t ngx_http_file_cache_reopen(ngx_http_request_t *r, ngx_http_cache_t *c); +static ngx_int_t ngx_http_file_cache_update_variant(ngx_http_request_t *r, + ngx_http_cache_t *c); static void ngx_http_file_cache_cleanup(void *data); static time_t ngx_http_file_cache_forced_expire(ngx_http_file_cache_t *cache); static time_t ngx_http_file_cache_expire(ngx_http_file_cache_t *cache); @@ -47,6 +49,8 @@ ngx_str_t *path); static ngx_int_t ngx_http_file_cache_manage_file(ngx_tree_ctx_t *ctx, ngx_str_t *path); +static ngx_int_t ngx_http_file_cache_manage_directory(ngx_tree_ctx_t *ctx, + ngx_str_t *path); static ngx_int_t ngx_http_file_cache_add_file(ngx_tree_ctx_t *ctx, ngx_str_t *path); static ngx_int_t ngx_http_file_cache_add(ngx_http_file_cache_t *cache, @@ -177,8 +181,6 @@ c->file.log = r->connection->log; c->file.fd = NGX_INVALID_FILE; - c->last_modified = -1; - return NGX_OK; } @@ -254,7 +256,7 @@ ngx_http_file_cache_open(ngx_http_request_t *r) { ngx_int_t rc, rv; - ngx_uint_t cold, test; + ngx_uint_t test; ngx_http_cache_t *c; ngx_pool_cleanup_t *cln; ngx_open_file_info_t of; @@ -296,8 +298,6 @@ return NGX_HTTP_CACHE_SCARCE; } - cold = cache->sh->cold; - if (rc == NGX_OK) { if (c->error) { @@ -310,18 +310,18 @@ } else { /* rc == NGX_DECLINED */ + test = cache->sh->cold ? 1 : 0; + if (c->min_uses > 1) { - if (!cold) { + if (!test) { return NGX_HTTP_CACHE_SCARCE; } - test = 1; rv = NGX_HTTP_CACHE_SCARCE; } else { c->temp_file = 1; - test = cold ? 1 : 0; rv = NGX_DECLINED; } } @@ -646,7 +646,7 @@ clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); - if (!clcf->aio) { + if (clcf->aio != NGX_HTTP_AIO_ON) { goto noaio; } @@ -1122,7 +1122,7 @@ } -void +ngx_int_t ngx_http_file_cache_set_header(ngx_http_request_t *r, u_char *buf) { ngx_http_file_cache_header_t *h = (ngx_http_file_cache_header_t *) buf; @@ -1164,9 +1164,10 @@ ngx_http_file_cache_vary(r, c->vary.data, c->vary.len, c->variant); ngx_memcpy(h->variant, c->variant, NGX_HTTP_CACHE_KEY_LEN); + } - } else { - ngx_memzero(c->variant, NGX_HTTP_CACHE_KEY_LEN); + if (ngx_http_file_cache_update_variant(r, c) != NGX_OK) { + return NGX_ERROR; } p = buf + sizeof(ngx_http_file_cache_header_t); @@ -1179,6 +1180,57 @@ } *p = LF; + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_file_cache_update_variant(ngx_http_request_t *r, ngx_http_cache_t *c) +{ + ngx_http_file_cache_t *cache; + + if (!c->secondary) { + return NGX_OK; + } + + if (c->vary.len + && ngx_memcmp(c->variant, c->key, NGX_HTTP_CACHE_KEY_LEN) == 0) + { + return NGX_OK; + } + + /* + * if the variant hash doesn't match one we used as a secondary + * cache key, switch back to the original key + */ + + cache = c->file_cache; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http file cache main key"); + + ngx_shmtx_lock(&cache->shpool->mutex); + + c->node->count--; + c->node->updating = 0; + c->node = NULL; + + ngx_shmtx_unlock(&cache->shpool->mutex); + + c->file.name.len = 0; + + ngx_memcpy(c->key, c->main, NGX_HTTP_CACHE_KEY_LEN); + + if (ngx_http_file_cache_exists(cache, c) == NGX_ERROR) { + return NGX_ERROR; + } + + if (ngx_http_file_cache_name(r, cache->path) != NGX_OK) { + return NGX_ERROR; + } + + return NGX_OK; } @@ -1204,38 +1256,6 @@ cache = c->file_cache; - if (c->secondary - && ngx_memcmp(c->variant, c->key, NGX_HTTP_CACHE_KEY_LEN) != 0) - { - /* - * if the variant hash doesn't match one we used as a secondary - * cache key, switch back to the original key - */ - - ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, - "http file cache main key"); - - ngx_shmtx_lock(&cache->shpool->mutex); - - c->node->count--; - c->node->updating = 0; - c->node = NULL; - - ngx_shmtx_unlock(&cache->shpool->mutex); - - c->file.name.len = 0; - - ngx_memcpy(c->key, c->main, NGX_HTTP_CACHE_KEY_LEN); - - if (ngx_http_file_cache_exists(cache, c) == NGX_ERROR) { - return; - } - - if (ngx_http_file_cache_name(r, cache->path) != NGX_OK) { - return; - } - } - c->updated = 1; c->updating = 0; @@ -1823,7 +1843,7 @@ tree.init_handler = NULL; tree.file_handler = ngx_http_file_cache_manage_file; - tree.pre_tree_handler = ngx_http_file_cache_noop; + tree.pre_tree_handler = ngx_http_file_cache_manage_directory; tree.post_tree_handler = ngx_http_file_cache_noop; tree.spec_handler = ngx_http_file_cache_delete_file; tree.data = cache; @@ -1888,6 +1908,19 @@ } +static ngx_int_t +ngx_http_file_cache_manage_directory(ngx_tree_ctx_t *ctx, ngx_str_t *path) +{ + if (path->len >= 5 + && ngx_strncmp(path->data + path->len - 5, "/temp", 5) == 0) + { + return NGX_DECLINED; + } + + return NGX_OK; +} + + static void ngx_http_file_cache_loader_sleep(ngx_http_file_cache_t *cache) { @@ -2037,11 +2070,12 @@ off_t max_size; u_char *last, *p; time_t inactive; + size_t len; ssize_t size; ngx_str_t s, name, *value; ngx_int_t loader_files; ngx_msec_t loader_sleep, loader_threshold; - ngx_uint_t i, n; + ngx_uint_t i, n, use_temp_path; ngx_array_t *caches; ngx_http_file_cache_t *cache, **ce; @@ -2055,6 +2089,8 @@ return NGX_CONF_ERROR; } + use_temp_path = 1; + inactive = 600; loader_files = 100; loader_sleep = 50; @@ -2115,6 +2151,25 @@ return NGX_CONF_ERROR; } + if (ngx_strncmp(value[i].data, "use_temp_path=", 14) == 0) { + + if (ngx_strcmp(&value[i].data[14], "on") == 0) { + use_temp_path = 1; + + } else if (ngx_strcmp(&value[i].data[14], "off") == 0) { + use_temp_path = 0; + + } else { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid use_temp_path value \"%V\", " + "it must be \"on\" or \"off\"", + &value[i]); + return NGX_CONF_ERROR; + } + + continue; + } + if (ngx_strncmp(value[i].data, "keys_zone=", 10) == 0) { name.data = value[i].data + 10; @@ -2237,6 +2292,37 @@ return NGX_CONF_ERROR; } + if (!use_temp_path) { + cache->temp_path = ngx_pcalloc(cf->pool, sizeof(ngx_path_t)); + if (cache->temp_path == NULL) { + return NGX_CONF_ERROR; + } + + len = cache->path->name.len + sizeof("/temp") - 1; + + p = ngx_pnalloc(cf->pool, len + 1); + if (p == NULL) { + return NGX_CONF_ERROR; + } + + cache->temp_path->name.len = len; + cache->temp_path->name.data = p; + + p = ngx_cpymem(p, cache->path->name.data, cache->path->name.len); + ngx_memcpy(p, "/temp", sizeof("/temp")); + + ngx_memcpy(&cache->temp_path->level, &cache->path->level, + 3 * sizeof(size_t)); + + cache->temp_path->len = cache->path->len; + cache->temp_path->conf_file = cf->conf_file->file.name.data; + cache->temp_path->line = cf->conf_file->line; + + if (ngx_add_path(cf, &cache->temp_path) != NGX_OK) { + return NGX_CONF_ERROR; + } + } + cache->shm_zone = ngx_shared_memory_add(cf, &name, size, cmd->post); if (cache->shm_zone == NULL) { return NGX_CONF_ERROR;

@@ -2155,6 +2155,10 @@ goto invalid; case sw_chunk_size: + if (ctx->size > NGX_MAX_OFF_T_VALUE / 16) { + goto invalid; + } + if (ch >= '0' && ch <= '9') { ctx->size = ctx->size * 16 + (ch - '0'); break; @@ -2304,6 +2308,10 @@ ctx->state = state; b->pos = pos; + if (ctx->size > NGX_MAX_OFF_T_VALUE - 5) { + goto invalid; + } + switch (state) { case sw_chunk_start: @@ -2340,10 +2348,6 @@ } - if (ctx->size < 0 || ctx->length < 0) { - goto invalid; - } - return rc; done:

@@ -652,6 +652,7 @@ if (n == -1) { if (err == NGX_EAGAIN) { + rev->ready = 0; if (!rev->timer_set) { ngx_add_timer(rev, c->listening->post_accept_timeout); @@ -2524,6 +2525,11 @@ return; } + if (r->reading_body) { + r->keepalive = 0; + r->lingering_close = 1; + } + if (!ngx_terminate && !ngx_exiting && r->keepalive

@@ -473,6 +473,7 @@ unsigned request_body_in_clean_file:1; unsigned request_body_file_group_access:1; unsigned request_body_file_log_level:3; + unsigned request_body_no_buffering:1; unsigned subrequest_in_memory:1; unsigned waited:1; @@ -509,6 +510,7 @@ unsigned keepalive:1; unsigned lingering_close:1; unsigned discard_body:1; + unsigned reading_body:1; unsigned internal:1; unsigned error_page:1; unsigned filter_finalize:1; @@ -574,12 +576,12 @@ typedef struct { ngx_http_posted_request_t terminal_posted_request; -#if (NGX_HAVE_AIO_SENDFILE) - u_char aio_preload; -#endif } ngx_http_ephemeral_t; +#define ngx_http_ephemeral(r) (void *) (&r->uri_start) + + extern ngx_http_header_t ngx_http_headers_in[]; extern ngx_http_header_out_t ngx_http_headers_out[];

@@ -24,8 +24,6 @@ ngx_chain_t *in); static ngx_int_t ngx_http_request_body_chunked_filter(ngx_http_request_t *r, ngx_chain_t *in); -static ngx_int_t ngx_http_request_body_save_filter(ngx_http_request_t *r, - ngx_chain_t *in); ngx_int_t @@ -44,12 +42,14 @@ #if (NGX_HTTP_SPDY) if (r->spdy_stream && r == r->main) { + r->request_body_no_buffering = 0; rc = ngx_http_spdy_read_request_body(r, post_handler); goto done; } #endif if (r != r->main || r->request_body || r->discard_body) { + r->request_body_no_buffering = 0; post_handler(r); return NGX_OK; } @@ -59,6 +59,10 @@ goto done; } + if (r->request_body_no_buffering) { + r->request_body_in_file_only = 0; + } + rb = ngx_pcalloc(r->pool, sizeof(ngx_http_request_body_t)); if (rb == NULL) { rc = NGX_HTTP_INTERNAL_SERVER_ERROR; @@ -81,6 +85,7 @@ r->request_body = rb; if (r->headers_in.content_length_n < 0 && !r->headers_in.chunked) { + r->request_body_no_buffering = 0; post_handler(r); return NGX_OK; } @@ -173,6 +178,8 @@ } } + r->request_body_no_buffering = 0; + post_handler(r); return NGX_OK; @@ -216,6 +223,21 @@ done: + if (r->request_body_no_buffering + && (rc == NGX_OK || rc == NGX_AGAIN)) + { + if (rc == NGX_OK) { + r->request_body_no_buffering = 0; + + } else { + /* rc == NGX_AGAIN */ + r->reading_body = 1; + } + + r->read_event_handler = ngx_http_block_reading; + post_handler(r); + } + if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { r->main->count--; } @@ -224,6 +246,26 @@ } +ngx_int_t +ngx_http_read_unbuffered_request_body(ngx_http_request_t *r) +{ + ngx_int_t rc; + + if (r->connection->read->timedout) { + r->connection->timedout = 1; + return NGX_HTTP_REQUEST_TIME_OUT; + } + + rc = ngx_http_do_read_client_request_body(r); + + if (rc == NGX_OK) { + r->reading_body = 0; + } + + return rc; +} + + static void ngx_http_read_client_request_body_handler(ngx_http_request_t *r) { @@ -266,32 +308,43 @@ for ( ;; ) { if (rb->buf->last == rb->buf->end) { - /* pass buffer to request body filter chain */ + if (rb->buf->pos != rb->buf->last) { - out.buf = rb->buf; - out.next = NULL; + /* pass buffer to request body filter chain */ - rc = ngx_http_request_body_filter(r, &out); + out.buf = rb->buf; + out.next = NULL; - if (rc != NGX_OK) { - return rc; - } + rc = ngx_http_request_body_filter(r, &out); - /* write to file */ + if (rc != NGX_OK) { + return rc; + } - if (ngx_http_write_request_body(r) != NGX_OK) { - return NGX_HTTP_INTERNAL_SERVER_ERROR; - } + } else { - /* update chains */ + /* update chains */ - rc = ngx_http_request_body_filter(r, NULL); + rc = ngx_http_request_body_filter(r, NULL); - if (rc != NGX_OK) { - return rc; + if (rc != NGX_OK) { + return rc; + } } if (rb->busy != NULL) { + if (r->request_body_no_buffering) { + if (c->read->timer_set) { + ngx_del_timer(c->read); + } + + if (ngx_handle_read_event(c->read, 0) != NGX_OK) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + + return NGX_AGAIN; + } + return NGX_HTTP_INTERNAL_SERVER_ERROR; } @@ -358,6 +411,22 @@ } if (!c->read->ready) { + + if (r->request_body_no_buffering + && rb->buf->pos != rb->buf->last) + { + /* pass buffer to request body filter chain */ + + out.buf = rb->buf; + out.next = NULL; + + rc = ngx_http_request_body_filter(r, &out); + + if (rc != NGX_OK) { + return rc; + } + } + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); ngx_add_timer(c->read, clcf->client_body_timeout); @@ -403,9 +472,10 @@ } } - r->read_event_handler = ngx_http_block_reading; - - rb->post_handler(r); + if (!r->request_body_no_buffering) { + r->read_event_handler = ngx_http_block_reading; + rb->post_handler(r); + } return NGX_OK; } @@ -415,7 +485,7 @@ ngx_http_write_request_body(ngx_http_request_t *r) { ssize_t n; - ngx_chain_t *cl; + ngx_chain_t *cl, *ln; ngx_temp_file_t *tf; ngx_http_request_body_t *rb; ngx_http_core_loc_conf_t *clcf; @@ -478,8 +548,13 @@ /* mark all buffers as written */ - for (cl = rb->bufs; cl; cl = cl->next) { + for (cl = rb->bufs; cl; /* void */) { + cl->buf->pos = cl->buf->last; + + ln = cl; + cl = cl->next; + ngx_free_chain(r->pool, ln); } rb->bufs = NULL; @@ -892,7 +967,7 @@ ll = &tl->next; } - rc = ngx_http_request_body_save_filter(r, out); + rc = ngx_http_top_request_body_filter(r, out); ngx_chain_update_chains(r->pool, &rb->free, &rb->busy, &out, (ngx_buf_tag_t) &ngx_http_read_client_request_body); @@ -936,7 +1011,7 @@ ngx_log_debug7(NGX_LOG_DEBUG_EVENT, r->connection->log, 0, "http body chunked buf " - "t:%d f:%d %p, pos %p, size: %z file: %O, size: %z", + "t:%d f:%d %p, pos %p, size: %z file: %O, size: %O", cl->buf->temporary, cl->buf->in_file, cl->buf->start, cl->buf->pos, cl->buf->last - cl->buf->pos, @@ -1044,7 +1119,7 @@ } } - rc = ngx_http_request_body_save_filter(r, out); + rc = ngx_http_top_request_body_filter(r, out); ngx_chain_update_chains(r->pool, &rb->free, &rb->busy, &out, (ngx_buf_tag_t) &ngx_http_read_client_request_body); @@ -1053,7 +1128,7 @@ } -static ngx_int_t +ngx_int_t ngx_http_request_body_save_filter(ngx_http_request_t *r, ngx_chain_t *in) { #if (NGX_DEBUG) @@ -1068,7 +1143,7 @@ for (cl = rb->bufs; cl; cl = cl->next) { ngx_log_debug7(NGX_LOG_DEBUG_EVENT, r->connection->log, 0, "http body old buf t:%d f:%d %p, pos %p, size: %z " - "file: %O, size: %z", + "file: %O, size: %O", cl->buf->temporary, cl->buf->in_file, cl->buf->start, cl->buf->pos, cl->buf->last - cl->buf->pos, @@ -1079,7 +1154,7 @@ for (cl = in; cl; cl = cl->next) { ngx_log_debug7(NGX_LOG_DEBUG_EVENT, r->connection->log, 0, "http body new buf t:%d f:%d %p, pos %p, size: %z " - "file: %O, size: %z", + "file: %O, size: %O", cl->buf->temporary, cl->buf->in_file, cl->buf->start, cl->buf->pos, cl->buf->last - cl->buf->pos, @@ -1095,5 +1170,14 @@ return NGX_HTTP_INTERNAL_SERVER_ERROR; } + if (rb->rest > 0 + && rb->buf && rb->buf->last == rb->buf->end + && !r->request_body_no_buffering) + { + if (ngx_http_write_request_body(r) != NGX_OK) { + return NGX_HTTP_INTERNAL_SERVER_ERROR; + } + } + return NGX_OK; }

@@ -1353,7 +1353,7 @@ pos += NGX_SPDY_DELTA_SIZE; ngx_log_debug2(NGX_LOG_DEBUG_HTTP, sc->connection->log, 0, - "spdy WINDOW_UPDATE sid:%ui delta:%ui", sid, delta); + "spdy WINDOW_UPDATE sid:%ui delta:%uz", sid, delta); if (sid) { stream = ngx_http_spdy_get_stream_by_id(sc, sid);

@@ -36,9 +36,12 @@ static ngx_int_t ngx_http_upstream_reinit(ngx_http_request_t *r, ngx_http_upstream_t *u); static void ngx_http_upstream_send_request(ngx_http_request_t *r, - ngx_http_upstream_t *u); + ngx_http_upstream_t *u, ngx_uint_t do_write); +static ngx_int_t ngx_http_upstream_send_request_body(ngx_http_request_t *r, + ngx_http_upstream_t *u, ngx_uint_t do_write); static void ngx_http_upstream_send_request_handler(ngx_http_request_t *r, ngx_http_upstream_t *u); +static void ngx_http_upstream_read_request_handler(ngx_http_request_t *r); static void ngx_http_upstream_process_header(ngx_http_request_t *r, ngx_http_upstream_t *u); static ngx_int_t ngx_http_upstream_test_next(ngx_http_request_t *r, @@ -76,7 +79,8 @@ static void ngx_http_upstream_process_downstream(ngx_http_request_t *r); static void ngx_http_upstream_process_upstream(ngx_http_request_t *r, ngx_http_upstream_t *u); -static void ngx_http_upstream_process_request(ngx_http_request_t *r); +static void ngx_http_upstream_process_request(ngx_http_request_t *r, + ngx_http_upstream_t *u); static void ngx_http_upstream_store(ngx_http_request_t *r, ngx_http_upstream_t *u); static void ngx_http_upstream_dummy_handler(ngx_http_request_t *r, @@ -359,6 +363,10 @@ ngx_http_upstream_status_variable, 0, NGX_HTTP_VAR_NOCACHEABLE, 0 }, + { ngx_string("upstream_header_time"), NULL, + ngx_http_upstream_response_time_variable, 1, + NGX_HTTP_VAR_NOCACHEABLE, 0 }, + { ngx_string("upstream_response_time"), NULL, ngx_http_upstream_response_time_variable, 0, NGX_HTTP_VAR_NOCACHEABLE, 0 }, @@ -441,9 +449,6 @@ u->peer.log = r->connection->log; u->peer.log_error = NGX_ERROR_ERR; -#if (NGX_THREADS) - u->peer.lock = &r->connection->lock; -#endif #if (NGX_HTTP_CACHE) r->cache = NULL; @@ -566,8 +571,11 @@ u->output.pool = r->pool; u->output.bufs.num = 1; u->output.bufs.size = clcf->client_body_buffer_size; - u->output.output_filter = ngx_chain_writer; - u->output.filter_ctx = &u->writer; + + if (u->output.output_filter == NULL) { + u->output.output_filter = ngx_chain_writer; + u->output.filter_ctx = &u->writer; + } u->writer.pool = r->pool; @@ -1315,6 +1323,7 @@ tp = ngx_timeofday(); u->state->response_sec = tp->sec; u->state->response_msec = tp->msec; + u->state->header_sec = (time_t) NGX_ERROR; rc = ngx_event_connect_peer(&u->peer); @@ -1429,7 +1438,7 @@ #endif - ngx_http_upstream_send_request(r, u); + ngx_http_upstream_send_request(r, u, 1); } @@ -1533,7 +1542,7 @@ c = r->connection; - ngx_http_upstream_send_request(r, u); + ngx_http_upstream_send_request(r, u, 1); ngx_http_run_posted_requests(c); return; @@ -1721,7 +1730,8 @@ static void -ngx_http_upstream_send_request(ngx_http_request_t *r, ngx_http_upstream_t *u) +ngx_http_upstream_send_request(ngx_http_request_t *r, ngx_http_upstream_t *u, + ngx_uint_t do_write) { ngx_int_t rc; ngx_connection_t *c; @@ -1738,21 +1748,25 @@ c->log->action = "sending request to upstream"; - rc = ngx_output_chain(&u->output, u->request_sent ? NULL : u->request_bufs); - - u->request_sent = 1; + rc = ngx_http_upstream_send_request_body(r, u, do_write); if (rc == NGX_ERROR) { ngx_http_upstream_next(r, u, NGX_HTTP_UPSTREAM_FT_ERROR); return; } - if (c->write->timer_set) { - ngx_del_timer(c->write); + if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { + ngx_http_upstream_finalize_request(r, u, rc); + return; } if (rc == NGX_AGAIN) { - ngx_add_timer(c->write, u->conf->send_timeout); + if (!c->write->ready) { + ngx_add_timer(c->write, u->conf->send_timeout); + + } else if (c->write->timer_set) { + ngx_del_timer(c->write); + } if (ngx_handle_write_event(c->write, u->conf->send_lowat) != NGX_OK) { ngx_http_upstream_finalize_request(r, u, @@ -1765,6 +1779,10 @@ /* rc == NGX_OK */ + if (c->write->timer_set) { + ngx_del_timer(c->write); + } + if (c->tcp_nopush == NGX_TCP_NOPUSH_SET) { if (ngx_tcp_push(c->fd) == NGX_ERROR) { ngx_log_error(NGX_LOG_CRIT, c->log, ngx_socket_errno, @@ -1794,6 +1812,123 @@ } +static ngx_int_t +ngx_http_upstream_send_request_body(ngx_http_request_t *r, + ngx_http_upstream_t *u, ngx_uint_t do_write) +{ + int tcp_nodelay; + ngx_int_t rc; + ngx_chain_t *out, *cl, *ln; + ngx_connection_t *c; + ngx_http_core_loc_conf_t *clcf; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http upstream send request body"); + + if (!r->request_body_no_buffering) { + + /* buffered request body */ + + if (!u->request_sent) { + u->request_sent = 1; + out = u->request_bufs; + + } else { + out = NULL; + } + + return ngx_output_chain(&u->output, out); + } + + if (!u->request_sent) { + u->request_sent = 1; + out = u->request_bufs; + + if (r->request_body->bufs) { + for (cl = out; cl->next; cl = out->next) { /* void */ } + cl->next = r->request_body->bufs; + r->request_body->bufs = NULL; + } + + c = u->peer.connection; + clcf = ngx_http_get_module_loc_conf(r, ngx_http_core_module); + + if (clcf->tcp_nodelay && c->tcp_nodelay == NGX_TCP_NODELAY_UNSET) { + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, c->log, 0, "tcp_nodelay"); + + tcp_nodelay = 1; + + if (setsockopt(c->fd, IPPROTO_TCP, TCP_NODELAY, + (const void *) &tcp_nodelay, sizeof(int)) == -1) + { + ngx_connection_error(c, ngx_socket_errno, + "setsockopt(TCP_NODELAY) failed"); + return NGX_ERROR; + } + + c->tcp_nodelay = NGX_TCP_NODELAY_SET; + } + + r->read_event_handler = ngx_http_upstream_read_request_handler; + + } else { + out = NULL; + } + + for ( ;; ) { + + if (do_write) { + rc = ngx_output_chain(&u->output, out); + + if (rc == NGX_ERROR) { + return NGX_ERROR; + } + + while (out) { + ln = out; + out = out->next; + ngx_free_chain(r->pool, ln); + } + + if (rc == NGX_OK && !r->reading_body) { + break; + } + } + + if (r->reading_body) { + /* read client request body */ + + rc = ngx_http_read_unbuffered_request_body(r); + + if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { + return rc; + } + + out = r->request_body->bufs; + r->request_body->bufs = NULL; + } + + /* stop if there is nothing to send */ + + if (out == NULL) { + rc = NGX_AGAIN; + break; + } + + do_write = 1; + } + + if (!r->reading_body) { + if (!u->store && !r->post_action && !u->conf->ignore_client_abort) { + r->read_event_handler = + ngx_http_upstream_rd_check_broken_connection; + } + } + + return rc; +} + + static void ngx_http_upstream_send_request_handler(ngx_http_request_t *r, ngx_http_upstream_t *u) @@ -1827,7 +1962,29 @@ return; } - ngx_http_upstream_send_request(r, u); + ngx_http_upstream_send_request(r, u, 1); +} + + +static void +ngx_http_upstream_read_request_handler(ngx_http_request_t *r) +{ + ngx_connection_t *c; + ngx_http_upstream_t *u; + + c = r->connection; + u = r->upstream; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http upstream read request handler"); + + if (c->read->timedout) { + c->timedout = 1; + ngx_http_upstream_finalize_request(r, u, NGX_HTTP_REQUEST_TIME_OUT); + return; + } + + ngx_http_upstream_send_request(r, u, 0); } @@ -1836,6 +1993,7 @@ { ssize_t n; ngx_int_t rc; + ngx_time_t *tp; ngx_connection_t *c; c = u->peer.connection; @@ -1956,6 +2114,10 @@ /* rc == NGX_OK */ + tp = ngx_timeofday(); + u->state->header_sec = tp->sec - u->state->response_sec; + u->state->header_msec = tp->msec - u->state->response_msec; + if (u->headers_in.status_n >= NGX_HTTP_SPECIAL_RESPONSE) { if (ngx_http_upstream_test_next(r, u) == NGX_OK) { @@ -2625,10 +2787,20 @@ if (u->headers_in.etag) { r->cache->etag = u->headers_in.etag->value; + + } else { + ngx_str_null(&r->cache->etag); } + + } else { + r->cache->last_modified = -1; + ngx_str_null(&r->cache->etag); } - ngx_http_file_cache_set_header(r, u->buffer.start); + if (ngx_http_file_cache_set_header(r, u->buffer.start) != NGX_OK) { + ngx_http_upstream_finalize_request(r, u, NGX_ERROR); + return; + } } else { u->cacheable = 0; @@ -2674,6 +2846,12 @@ if (p->cacheable) { p->temp_file->persistent = 1; +#if (NGX_HTTP_CACHE) + if (r->cache && r->cache->file_cache->temp_path) { + p->temp_file->path = r->cache->file_cache->temp_path; + } +#endif + } else { p->temp_file->log_level = NGX_LOG_WARN; p->temp_file->warn = "an upstream response is buffered " @@ -3323,7 +3501,7 @@ } } - ngx_http_upstream_process_request(r); + ngx_http_upstream_process_request(r, u); } @@ -3391,18 +3569,17 @@ } } - ngx_http_upstream_process_request(r); + ngx_http_upstream_process_request(r, u); } static void -ngx_http_upstream_process_request(ngx_http_request_t *r) +ngx_http_upstream_process_request(ngx_http_request_t *r, + ngx_http_upstream_t *u) { - ngx_temp_file_t *tf; - ngx_event_pipe_t *p; - ngx_http_upstream_t *u; + ngx_temp_file_t *tf; + ngx_event_pipe_t *p; - u = r->upstream; p = u->pipe; if (u->peer.connection) { @@ -3603,7 +3780,9 @@ "upstream timed out"); } - if (u->peer.cached && ft_type == NGX_HTTP_UPSTREAM_FT_ERROR) { + if (u->peer.cached && ft_type == NGX_HTTP_UPSTREAM_FT_ERROR + && (!u->request_sent || !r->request_body_no_buffering)) + { status = 0; /* TODO: inform balancer instead */ @@ -3651,6 +3830,7 @@ if (u->peer.tries == 0 || !(u->conf->next_upstream & ft_type) + || (u->request_sent && r->request_body_no_buffering) || (timeout && ngx_current_msec - u->peer.start_time >= timeout)) { #if (NGX_HTTP_CACHE) @@ -3725,11 +3905,15 @@ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, "finalize http upstream request: %i", rc); - if (u->cleanup) { - *u->cleanup = NULL; - u->cleanup = NULL; + if (u->cleanup == NULL) { + /* the request was already finalized */ + ngx_http_finalize_request(r, NGX_DONE); + return; } + *u->cleanup = NULL; + u->cleanup = NULL; + if (u->resolved && u->resolved->ctx) { ngx_resolve_name_done(u->resolved->ctx); u->resolved->ctx = NULL; @@ -4811,8 +4995,18 @@ for ( ;; ) { if (state[i].status) { - ms = (ngx_msec_int_t) - (state[i].response_sec * 1000 + state[i].response_msec); + + if (data + && state[i].header_sec != (time_t) NGX_ERROR) + { + ms = (ngx_msec_int_t) + (state[i].header_sec * 1000 + state[i].header_msec); + + } else { + ms = (ngx_msec_int_t) + (state[i].response_sec * 1000 + state[i].response_msec); + } + ms = ngx_max(ms, 0); p = ngx_sprintf(p, "%T.%03M", (time_t) ms / 1000, ms % 1000); @@ -5379,7 +5573,7 @@ uscf->default_port = u->default_port; uscf->no_port = u->no_port; - if (u->naddrs == 1) { + if (u->naddrs == 1 && (u->port || u->family == AF_UNIX)) { uscf->servers = ngx_array_create(cf->pool, 1, sizeof(ngx_http_upstream_server_t)); if (uscf->servers == NULL) {

@@ -60,6 +60,8 @@ ngx_uint_t status; time_t response_sec; ngx_uint_t response_msec; + time_t header_sec; + ngx_uint_t header_msec; off_t response_length; ngx_str_t *peer; @@ -158,6 +160,7 @@ ngx_uint_t store_access; ngx_uint_t next_upstream_tries; ngx_flag_t buffering; + ngx_flag_t request_buffering; ngx_flag_t pass_request_headers; ngx_flag_t pass_request_body;

@@ -44,8 +44,6 @@ struct ngx_http_upstream_rr_peers_s { ngx_uint_t number; - /* ngx_mutex_t *mutex; */ - ngx_uint_t total_weight; unsigned single:1;

@@ -1081,6 +1081,10 @@ v->no_cacheable = 0; v->not_found = 0; + } else if (r->reading_body) { + v->not_found = 1; + v->no_cacheable = 1; + } else if (r->headers_in.content_length_n >= 0) { p = ngx_pnalloc(r->pool, NGX_OFF_T_LEN); if (p == NULL) {

@@ -73,7 +73,7 @@ ngx_log_debug7(NGX_LOG_DEBUG_EVENT, c->log, 0, "write old buf t:%d f:%d %p, pos %p, size: %z " - "file: %O, size: %z", + "file: %O, size: %O", cl->buf->temporary, cl->buf->in_file, cl->buf->start, cl->buf->pos, cl->buf->last - cl->buf->pos, @@ -129,7 +129,7 @@ ngx_log_debug7(NGX_LOG_DEBUG_EVENT, c->log, 0, "write new buf t:%d f:%d %p, pos %p, size: %z " - "file: %O, size: %z", + "file: %O, size: %O", cl->buf->temporary, cl->buf->in_file, cl->buf->start, cl->buf->pos, cl->buf->last - cl->buf->pos,

@@ -98,7 +98,7 @@ *(ngx_mail_conf_ctx_t **) conf = ctx; - /* count the number of the http modules and set up their indices */ + /* count the number of the mail modules and set up their indices */ ngx_mail_max_module = 0; for (m = 0; ngx_modules[m]; m++) {

@@ -336,6 +336,8 @@ ngx_mail_auth_state_pt auth_state; ngx_str_t internal_server_error; + ngx_str_t cert_error; + ngx_str_t no_cert; };

@@ -16,6 +16,7 @@ ngx_addr_t *peer; ngx_msec_t timeout; + ngx_flag_t pass_client_cert; ngx_str_t host_header; ngx_str_t uri; @@ -106,6 +107,13 @@ 0, NULL }, + { ngx_string("auth_http_pass_client_cert"), + NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_MAIL_SRV_CONF_OFFSET, + offsetof(ngx_mail_auth_http_conf_t, pass_client_cert), + NULL }, + ngx_null_command }; @@ -1143,6 +1151,12 @@ size_t len; ngx_buf_t *b; ngx_str_t login, passwd; +#if (NGX_MAIL_SSL) + ngx_str_t verify, subject, issuer, serial, fingerprint, + raw_cert, cert; + ngx_connection_t *c; + ngx_mail_ssl_conf_t *sslcf; +#endif ngx_mail_core_srv_conf_t *cscf; if (ngx_mail_auth_http_escape(pool, &s->login, &login) != NGX_OK) { @@ -1153,6 +1167,62 @@ return NULL; } +#if (NGX_MAIL_SSL) + + c = s->connection; + sslcf = ngx_mail_get_module_srv_conf(s, ngx_mail_ssl_module); + + if (c->ssl && sslcf->verify) { + + /* certificate details */ + + if (ngx_ssl_get_client_verify(c, pool, &verify) != NGX_OK) { + return NULL; + } + + if (ngx_ssl_get_subject_dn(c, pool, &subject) != NGX_OK) { + return NULL; + } + + if (ngx_ssl_get_issuer_dn(c, pool, &issuer) != NGX_OK) { + return NULL; + } + + if (ngx_ssl_get_serial_number(c, pool, &serial) != NGX_OK) { + return NULL; + } + + if (ngx_ssl_get_fingerprint(c, pool, &fingerprint) != NGX_OK) { + return NULL; + } + + if (ahcf->pass_client_cert) { + + /* certificate itself, if configured */ + + if (ngx_ssl_get_raw_certificate(c, pool, &raw_cert) != NGX_OK) { + return NULL; + } + + if (ngx_mail_auth_http_escape(pool, &raw_cert, &cert) != NGX_OK) { + return NULL; + } + + } else { + ngx_str_null(&cert); + } + + } else { + ngx_str_null(&verify); + ngx_str_null(&subject); + ngx_str_null(&issuer); + ngx_str_null(&serial); + ngx_str_null(&fingerprint); + ngx_str_null(&cert); + } + +#endif + cscf = ngx_mail_get_module_srv_conf(s, ngx_mail_core_module); len = sizeof("GET ") - 1 + ahcf->uri.len + sizeof(" HTTP/1.0" CRLF) - 1 @@ -1170,9 +1240,19 @@ + sizeof("Client-IP: ") - 1 + s->connection->addr_text.len + sizeof(CRLF) - 1 + sizeof("Client-Host: ") - 1 + s->host.len + sizeof(CRLF) - 1 - + sizeof("Auth-SMTP-Helo: ") - 1 + s->smtp_helo.len - + sizeof("Auth-SMTP-From: ") - 1 + s->smtp_from.len - + sizeof("Auth-SMTP-To: ") - 1 + s->smtp_to.len + + sizeof("Auth-SMTP-Helo: ") - 1 + s->smtp_helo.len + sizeof(CRLF) - 1 + + sizeof("Auth-SMTP-From: ") - 1 + s->smtp_from.len + sizeof(CRLF) - 1 + + sizeof("Auth-SMTP-To: ") - 1 + s->smtp_to.len + sizeof(CRLF) - 1 +#if (NGX_MAIL_SSL) + + sizeof("Auth-SSL: on" CRLF) - 1 + + sizeof("Auth-SSL-Verify: ") - 1 + verify.len + sizeof(CRLF) - 1 + + sizeof("Auth-SSL-Subject: ") - 1 + subject.len + sizeof(CRLF) - 1 + + sizeof("Auth-SSL-Issuer: ") - 1 + issuer.len + sizeof(CRLF) - 1 + + sizeof("Auth-SSL-Serial: ") - 1 + serial.len + sizeof(CRLF) - 1 + + sizeof("Auth-SSL-Fingerprint: ") - 1 + fingerprint.len + + sizeof(CRLF) - 1 + + sizeof("Auth-SSL-Cert: ") - 1 + cert.len + sizeof(CRLF) - 1 +#endif + ahcf->header.len + sizeof(CRLF) - 1; @@ -1255,6 +1335,57 @@ } +#if (NGX_MAIL_SSL) + + if (c->ssl) { + b->last = ngx_cpymem(b->last, "Auth-SSL: on" CRLF, + sizeof("Auth-SSL: on" CRLF) - 1); + + if (verify.len) { + b->last = ngx_cpymem(b->last, "Auth-SSL-Verify: ", + sizeof("Auth-SSL-Verify: ") - 1); + b->last = ngx_copy(b->last, verify.data, verify.len); + *b->last++ = CR; *b->last++ = LF; + } + + if (subject.len) { + b->last = ngx_cpymem(b->last, "Auth-SSL-Subject: ", + sizeof("Auth-SSL-Subject: ") - 1); + b->last = ngx_copy(b->last, subject.data, subject.len); + *b->last++ = CR; *b->last++ = LF; + } + + if (issuer.len) { + b->last = ngx_cpymem(b->last, "Auth-SSL-Issuer: ", + sizeof("Auth-SSL-Issuer: ") - 1); + b->last = ngx_copy(b->last, issuer.data, issuer.len); + *b->last++ = CR; *b->last++ = LF; + } + + if (serial.len) { + b->last = ngx_cpymem(b->last, "Auth-SSL-Serial: ", + sizeof("Auth-SSL-Serial: ") - 1); + b->last = ngx_copy(b->last, serial.data, serial.len); + *b->last++ = CR; *b->last++ = LF; + } + + if (fingerprint.len) { + b->last = ngx_cpymem(b->last, "Auth-SSL-Fingerprint: ", + sizeof("Auth-SSL-Fingerprint: ") - 1); + b->last = ngx_copy(b->last, fingerprint.data, fingerprint.len); + *b->last++ = CR; *b->last++ = LF; + } + + if (cert.len) { + b->last = ngx_cpymem(b->last, "Auth-SSL-Cert: ", + sizeof("Auth-SSL-Cert: ") - 1); + b->last = ngx_copy(b->last, cert.data, cert.len); + *b->last++ = CR; *b->last++ = LF; + } + } + +#endif + if (ahcf->header.len) { b->last = ngx_copy(b->last, ahcf->header.data, ahcf->header.len); } @@ -1263,14 +1394,9 @@ *b->last++ = CR; *b->last++ = LF; #if (NGX_DEBUG_MAIL_PASSWD) - { - ngx_str_t l; - - l.len = b->last - b->pos; - l.data = b->pos; - ngx_log_debug1(NGX_LOG_DEBUG_MAIL, s->connection->log, 0, - "mail auth http header:%N\"%V\"", &l); - } + ngx_log_debug2(NGX_LOG_DEBUG_MAIL, s->connection->log, 0, + "mail auth http header:%N\"%*s\"", + (size_t) (b->last - b->pos), b->pos); #endif return b; @@ -1316,6 +1442,7 @@ } ahcf->timeout = NGX_CONF_UNSET_MSEC; + ahcf->pass_client_cert = NGX_CONF_UNSET; ahcf->file = cf->conf_file->file.name.data; ahcf->line = cf->conf_file->line; @@ -1351,6 +1478,8 @@ ngx_conf_merge_msec_value(conf->timeout, prev->timeout, 60000); + ngx_conf_merge_value(conf->pass_client_cert, prev->pass_client_cert, 0); + if (conf->headers == NULL) { conf->headers = prev->headers; conf->header = prev->header;

@@ -336,7 +336,7 @@ off = offsetof(struct sockaddr_in6, sin6_addr); len = 16; sin6 = (struct sockaddr_in6 *) sa; - port = sin6->sin6_port; + port = ntohs(sin6->sin6_port); break; #endif @@ -352,7 +352,7 @@ off = offsetof(struct sockaddr_in, sin_addr); len = 4; sin = (struct sockaddr_in *) sa; - port = sin->sin_port; + port = ntohs(sin->sin_port); break; }

@@ -16,6 +16,8 @@ #if (NGX_MAIL_SSL) static void ngx_mail_ssl_init_connection(ngx_ssl_t *ssl, ngx_connection_t *c); static void ngx_mail_ssl_handshake_handler(ngx_connection_t *c); +static ngx_int_t ngx_mail_verify_cert(ngx_mail_session_t *s, + ngx_connection_t *c); #endif @@ -247,6 +249,10 @@ s = c->data; + if (ngx_mail_verify_cert(s, c) != NGX_OK) { + return; + } + if (s->starttls) { cscf = ngx_mail_get_module_srv_conf(s, ngx_mail_core_module); @@ -267,6 +273,71 @@ ngx_mail_close_connection(c); } + +static ngx_int_t +ngx_mail_verify_cert(ngx_mail_session_t *s, ngx_connection_t *c) +{ + long rc; + X509 *cert; + ngx_mail_ssl_conf_t *sslcf; + ngx_mail_core_srv_conf_t *cscf; + + sslcf = ngx_mail_get_module_srv_conf(s, ngx_mail_ssl_module); + + if (!sslcf->verify) { + return NGX_OK; + } + + rc = SSL_get_verify_result(c->ssl->connection); + + if (rc != X509_V_OK + && (sslcf->verify != 3 || !ngx_ssl_verify_error_optional(rc))) + { + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "client SSL certificate verify error: (%l:%s)", + rc, X509_verify_cert_error_string(rc)); + + ngx_ssl_remove_cached_session(sslcf->ssl.ctx, + (SSL_get0_session(c->ssl->connection))); + + cscf = ngx_mail_get_module_srv_conf(s, ngx_mail_core_module); + + s->out = cscf->protocol->cert_error; + s->quit = 1; + + c->write->handler = ngx_mail_send; + + ngx_mail_send(s->connection->write); + return NGX_ERROR; + } + + if (sslcf->verify == 1) { + cert = SSL_get_peer_certificate(c->ssl->connection); + + if (cert == NULL) { + ngx_log_error(NGX_LOG_INFO, c->log, 0, + "client sent no required SSL certificate"); + + ngx_ssl_remove_cached_session(sslcf->ssl.ctx, + (SSL_get0_session(c->ssl->connection))); + + cscf = ngx_mail_get_module_srv_conf(s, ngx_mail_core_module); + + s->out = cscf->protocol->no_cert; + s->quit = 1; + + c->write->handler = ngx_mail_send; + + ngx_mail_send(s->connection->write); + return NGX_ERROR; + } + + X509_free(cert); + } + + return NGX_OK; +} + #endif

@@ -52,7 +52,9 @@ ngx_mail_imap_parse_command, ngx_mail_imap_auth_state, - ngx_string("* BAD internal server error" CRLF) + ngx_string("* BAD internal server error" CRLF), + ngx_string("* BYE SSL certificate error" CRLF), + ngx_string("* BYE No required SSL certificate" CRLF) };

@@ -58,7 +58,9 @@ ngx_mail_pop3_parse_command, ngx_mail_pop3_auth_state, - ngx_string("-ERR internal server error" CRLF) + ngx_string("-ERR internal server error" CRLF), + ngx_string("-ERR SSL certificate error" CRLF), + ngx_string("-ERR No required SSL certificate" CRLF) };

@@ -45,7 +45,9 @@ ngx_mail_smtp_parse_command, ngx_mail_smtp_auth_state, - ngx_string("451 4.3.2 Internal server error" CRLF) + ngx_string("451 4.3.2 Internal server error" CRLF), + ngx_string("421 4.7.1 SSL certificate error" CRLF), + ngx_string("421 4.7.1 No required SSL certificate" CRLF) };

@@ -46,6 +46,15 @@ }; +static ngx_conf_enum_t ngx_mail_ssl_verify[] = { + { ngx_string("off"), 0 }, + { ngx_string("on"), 1 }, + { ngx_string("optional"), 2 }, + { ngx_string("optional_no_ca"), 3 }, + { ngx_null_string, 0 } +}; + + static ngx_command_t ngx_mail_ssl_commands[] = { { ngx_string("ssl"), @@ -146,6 +155,41 @@ offsetof(ngx_mail_ssl_conf_t, session_timeout), NULL }, + { ngx_string("ssl_verify_client"), + NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_enum_slot, + NGX_MAIL_SRV_CONF_OFFSET, + offsetof(ngx_mail_ssl_conf_t, verify), + &ngx_mail_ssl_verify }, + + { ngx_string("ssl_verify_depth"), + NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_num_slot, + NGX_MAIL_SRV_CONF_OFFSET, + offsetof(ngx_mail_ssl_conf_t, verify_depth), + NULL }, + + { ngx_string("ssl_client_certificate"), + NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + NGX_MAIL_SRV_CONF_OFFSET, + offsetof(ngx_mail_ssl_conf_t, client_certificate), + NULL }, + + { ngx_string("ssl_trusted_certificate"), + NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + NGX_MAIL_SRV_CONF_OFFSET, + offsetof(ngx_mail_ssl_conf_t, trusted_certificate), + NULL }, + + { ngx_string("ssl_crl"), + NGX_MAIL_MAIN_CONF|NGX_MAIL_SRV_CONF|NGX_CONF_TAKE1, + ngx_conf_set_str_slot, + NGX_MAIL_SRV_CONF_OFFSET, + offsetof(ngx_mail_ssl_conf_t, crl), + NULL }, + ngx_null_command }; @@ -198,6 +242,9 @@ * scf->certificate_key = { 0, NULL }; * scf->dhparam = { 0, NULL }; * scf->ecdh_curve = { 0, NULL }; + * scf->client_certificate = { 0, NULL }; + * scf->trusted_certificate = { 0, NULL }; + * scf->crl = { 0, NULL }; * scf->ciphers = { 0, NULL }; * scf->shm_zone = NULL; */ @@ -206,6 +253,8 @@ scf->starttls = NGX_CONF_UNSET_UINT; scf->passwords = NGX_CONF_UNSET_PTR; scf->prefer_server_ciphers = NGX_CONF_UNSET; + scf->verify = NGX_CONF_UNSET_UINT; + scf->verify_depth = NGX_CONF_UNSET_UINT; scf->builtin_session_cache = NGX_CONF_UNSET; scf->session_timeout = NGX_CONF_UNSET; scf->session_tickets = NGX_CONF_UNSET; @@ -238,6 +287,9 @@ (NGX_CONF_BITMASK_SET|NGX_SSL_SSLv3|NGX_SSL_TLSv1 |NGX_SSL_TLSv1_1|NGX_SSL_TLSv1_2)); + ngx_conf_merge_uint_value(conf->verify, prev->verify, 0); + ngx_conf_merge_uint_value(conf->verify_depth, prev->verify_depth, 1); + ngx_conf_merge_str_value(conf->certificate, prev->certificate, ""); ngx_conf_merge_str_value(conf->certificate_key, prev->certificate_key, ""); @@ -248,6 +300,12 @@ ngx_conf_merge_str_value(conf->ecdh_curve, prev->ecdh_curve, NGX_DEFAULT_ECDH_CURVE); + ngx_conf_merge_str_value(conf->client_certificate, + prev->client_certificate, ""); + ngx_conf_merge_str_value(conf->trusted_certificate, + prev->trusted_certificate, ""); + ngx_conf_merge_str_value(conf->crl, prev->crl, ""); + ngx_conf_merge_str_value(conf->ciphers, prev->ciphers, NGX_DEFAULT_CIPHERS); @@ -320,6 +378,35 @@ return NGX_CONF_ERROR; } + if (conf->verify) { + + if (conf->client_certificate.len == 0 && conf->verify != 3) { + ngx_log_error(NGX_LOG_EMERG, cf->log, 0, + "no ssl_client_certificate for ssl_client_verify"); + return NGX_CONF_ERROR; + } + + if (ngx_ssl_client_certificate(cf, &conf->ssl, + &conf->client_certificate, + conf->verify_depth) + != NGX_OK) + { + return NGX_CONF_ERROR; + } + + if (ngx_ssl_trusted_certificate(cf, &conf->ssl, + &conf->trusted_certificate, + conf->verify_depth) + != NGX_OK) + { + return NGX_CONF_ERROR; + } + + if (ngx_ssl_crl(cf, &conf->ssl, &conf->crl) != NGX_OK) { + return NGX_CONF_ERROR; + } + } + if (SSL_CTX_set_cipher_list(conf->ssl.ctx, (const char *) conf->ciphers.data) == 0) @@ -334,7 +421,9 @@ SSL_CTX_set_options(conf->ssl.ctx, SSL_OP_CIPHER_SERVER_PREFERENCE); } +#ifndef LIBRESSL_VERSION_NUMBER SSL_CTX_set_tmp_rsa_callback(conf->ssl.ctx, ngx_ssl_rsa512_key_callback); +#endif if (ngx_ssl_dhparam(cf, &conf->ssl, &conf->dhparam) != NGX_OK) { return NGX_CONF_ERROR;

@@ -28,6 +28,9 @@ ngx_uint_t starttls; ngx_uint_t protocols; + ngx_uint_t verify; + ngx_uint_t verify_depth; + ssize_t builtin_session_cache; time_t session_timeout; @@ -36,6 +39,9 @@ ngx_str_t certificate_key; ngx_str_t dhparam; ngx_str_t ecdh_curve; + ngx_str_t client_certificate; + ngx_str_t trusted_certificate; + ngx_str_t crl; ngx_str_t ciphers;

@@ -111,6 +111,9 @@ } send += trailer.size; + + } else { + trailer.count = 0; } /*

@@ -36,6 +36,28 @@ static void ngx_file_aio_event_handler(ngx_event_t *ev); +ngx_int_t +ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool) +{ + ngx_event_aio_t *aio; + + aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t)); + if (aio == NULL) { + return NGX_ERROR; + } + + aio->file = file; + aio->fd = file->fd; + aio->event.data = aio; + aio->event.ready = 1; + aio->event.log = file->log; + + file->aio = aio; + + return NGX_OK; +} + + ssize_t ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset, ngx_pool_t *pool) @@ -48,25 +70,11 @@ return ngx_read_file(file, buf, size, offset); } - aio = file->aio; - - if (aio == NULL) { - aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t)); - if (aio == NULL) { - return NGX_ERROR; - } - - aio->file = file; - aio->fd = file->fd; - aio->event.data = aio; - aio->event.ready = 1; - aio->event.log = file->log; -#if (NGX_HAVE_AIO_SENDFILE) - aio->last_offset = -1; -#endif - file->aio = aio; + if (file->aio == NULL && ngx_file_aio_init(file, pool) != NGX_OK) { + return NGX_ERROR; } + aio = file->aio; ev = &aio->event; if (!ev->ready) {

@@ -9,6 +9,12 @@ #include <ngx_core.h> +#if (NGX_THREADS) +#include <ngx_thread_pool.h> +static void ngx_thread_read_handler(void *data, ngx_log_t *log); +#endif + + #if (NGX_HAVE_FILE_AIO) ngx_uint_t ngx_file_aio = 1; @@ -64,6 +70,109 @@ } +#if (NGX_THREADS) + +typedef struct { + ngx_fd_t fd; + u_char *buf; + size_t size; + off_t offset; + + size_t read; + ngx_err_t err; +} ngx_thread_read_ctx_t; + + +ssize_t +ngx_thread_read(ngx_thread_task_t **taskp, ngx_file_t *file, u_char *buf, + size_t size, off_t offset, ngx_pool_t *pool) +{ + ngx_thread_task_t *task; + ngx_thread_read_ctx_t *ctx; + + ngx_log_debug4(NGX_LOG_DEBUG_CORE, file->log, 0, + "thread read: %d, %p, %uz, %O", + file->fd, buf, size, offset); + + task = *taskp; + + if (task == NULL) { + task = ngx_thread_task_alloc(pool, sizeof(ngx_thread_read_ctx_t)); + if (task == NULL) { + return NGX_ERROR; + } + + task->handler = ngx_thread_read_handler; + + *taskp = task; + } + + ctx = task->ctx; + + if (task->event.complete) { + task->event.complete = 0; + + if (ctx->err) { + ngx_log_error(NGX_LOG_CRIT, file->log, ctx->err, + "pread() \"%s\" failed", file->name.data); + return NGX_ERROR; + } + + return ctx->read; + } + + ctx->fd = file->fd; + ctx->buf = buf; + ctx->size = size; + ctx->offset = offset; + + if (file->thread_handler(task, file) != NGX_OK) { + return NGX_ERROR; + } + + return NGX_AGAIN; +} + + +#if (NGX_HAVE_PREAD) + +static void +ngx_thread_read_handler(void *data, ngx_log_t *log) +{ + ngx_thread_read_ctx_t *ctx = data; + + ssize_t n; + + ngx_log_debug0(NGX_LOG_DEBUG_CORE, log, 0, "thread read handler"); + + n = pread(ctx->fd, ctx->buf, ctx->size, ctx->offset); + + if (n == -1) { + ctx->err = ngx_errno; + + } else { + ctx->read = n; + ctx->err = 0; + } + +#if 0 + ngx_time_update(); +#endif + + ngx_log_debug4(NGX_LOG_DEBUG_CORE, log, 0, + "pread: %z (err: %i) of %uz @%O", + n, ctx->err, ctx->size, ctx->offset); +} + +#else + +#error pread() is required! + +#endif + +#endif /* NGX_THREADS */ + + ssize_t ngx_write_file(ngx_file_t *file, u_char *buf, size_t size, off_t offset) {

@@ -375,6 +375,7 @@ #if (NGX_HAVE_FILE_AIO) +ngx_int_t ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool); ssize_t ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset, ngx_pool_t *pool); @@ -382,5 +383,10 @@ #endif +#if (NGX_THREADS) +ssize_t ngx_thread_read(ngx_thread_task_t **taskp, ngx_file_t *file, + u_char *buf, size_t size, off_t offset, ngx_pool_t *pool); +#endif + #endif /* _NGX_FILES_H_INCLUDED_ */

@@ -100,12 +100,6 @@ #endif -#if (__FreeBSD_version < 430000 || __FreeBSD_version < 500012) - -pid_t rfork_thread(int flags, void *stack, int (*func)(void *arg), void *arg); - -#endif - #ifndef IOV_MAX #define IOV_MAX 1024 #endif

@@ -32,19 +32,23 @@ ngx_chain_t * ngx_freebsd_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) { - int rc, flags; - off_t send, prev_send, sent; - size_t file_size; - ssize_t n; - ngx_uint_t eintr, eagain; - ngx_err_t err; - ngx_buf_t *file; - ngx_event_t *wev; - ngx_chain_t *cl; - ngx_iovec_t header, trailer; - struct sf_hdtr hdtr; - struct iovec headers[NGX_IOVS_PREALLOCATE]; - struct iovec trailers[NGX_IOVS_PREALLOCATE]; + int rc, flags; + off_t send, prev_send, sent; + size_t file_size; + ssize_t n; + ngx_uint_t eintr, eagain; + ngx_err_t err; + ngx_buf_t *file; + ngx_event_t *wev; + ngx_chain_t *cl; + ngx_iovec_t header, trailer; + struct sf_hdtr hdtr; + struct iovec headers[NGX_IOVS_PREALLOCATE]; + struct iovec trailers[NGX_IOVS_PREALLOCATE]; +#if (NGX_HAVE_AIO_SENDFILE) + ngx_uint_t ebusy; + ngx_event_aio_t *aio; +#endif wev = c->write; @@ -73,6 +77,11 @@ eagain = 0; flags = 0; +#if (NGX_HAVE_AIO_SENDFILE && NGX_SUPPRESS_WARN) + aio = NULL; + file = NULL; +#endif + header.iovs = headers; header.nalloc = NGX_IOVS_PREALLOCATE; @@ -81,6 +90,9 @@ for ( ;; ) { eintr = 0; +#if (NGX_HAVE_AIO_SENDFILE) + ebusy = 0; +#endif prev_send = send; /* create the header iovec and coalesce the neighbouring bufs */ @@ -160,7 +172,8 @@ sent = 0; #if (NGX_HAVE_AIO_SENDFILE) - flags = c->aio_sendfile ? SF_NODISKIO : 0; + aio = file->file->aio; + flags = (aio && aio->preload_handler) ? SF_NODISKIO : 0; #endif rc = sendfile(file->file->fd, c->fd, file->file_pos, @@ -180,7 +193,7 @@ #if (NGX_HAVE_AIO_SENDFILE) case NGX_EBUSY: - c->busy_sendfile = file; + ebusy = 1; break; #endif @@ -232,9 +245,41 @@ in = ngx_chain_update_sent(in, sent); #if (NGX_HAVE_AIO_SENDFILE) - if (c->busy_sendfile) { + + if (ebusy) { + if (sent == 0) { + c->busy_count++; + + if (c->busy_count > 2) { + ngx_log_error(NGX_LOG_ALERT, c->log, 0, + "sendfile(%V) returned busy again", + &file->file->name); + + c->busy_count = 0; + aio->preload_handler = NULL; + + send = prev_send; + continue; + } + + } else { + c->busy_count = 0; + } + + rc = aio->preload_handler(file); + + if (rc > 0) { + send = prev_send + sent; + continue; + } + return in; } + + if (flags == SF_NODISKIO) { + c->busy_count = 0; + } + #endif if (eagain) {

@@ -24,6 +24,28 @@ } +ngx_int_t +ngx_file_aio_init(ngx_file_t *file, ngx_pool_t *pool) +{ + ngx_event_aio_t *aio; + + aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t)); + if (aio == NULL) { + return NGX_ERROR; + } + + aio->file = file; + aio->fd = file->fd; + aio->event.data = aio; + aio->event.ready = 1; + aio->event.log = file->log; + + file->aio = aio; + + return NGX_OK; +} + + ssize_t ngx_file_aio_read(ngx_file_t *file, u_char *buf, size_t size, off_t offset, ngx_pool_t *pool) @@ -37,22 +59,11 @@ return ngx_read_file(file, buf, size, offset); } - aio = file->aio; - - if (aio == NULL) { - aio = ngx_pcalloc(pool, sizeof(ngx_event_aio_t)); - if (aio == NULL) { - return NGX_ERROR; - } - - aio->file = file; - aio->fd = file->fd; - aio->event.data = aio; - aio->event.ready = 1; - aio->event.log = file->log; - file->aio = aio; + if (file->aio == NULL && ngx_file_aio_init(file, pool) != NGX_OK) { + return NGX_ERROR; } + aio = file->aio; ev = &aio->event; if (!ev->ready) {

@@ -93,11 +93,11 @@ #endif -#if (NGX_HAVE_FILE_AIO) #if (NGX_HAVE_SYS_EVENTFD_H) #include <sys/eventfd.h> #endif #include <sys/syscall.h> +#if (NGX_HAVE_FILE_AIO) #include <linux/aio_abi.h> typedef struct iocb ngx_aiocb_t; #endif

@@ -10,6 +10,22 @@ #include <ngx_event.h> +static ssize_t ngx_linux_sendfile(ngx_connection_t *c, ngx_buf_t *file, + size_t size); + +#if (NGX_THREADS) +#include <ngx_thread_pool.h> + +#if !(NGX_HAVE_SENDFILE64) +#error sendfile64() is required! +#endif + +static ngx_int_t ngx_linux_sendfile_thread(ngx_connection_t *c, ngx_buf_t *file, + size_t size, size_t *sent); +static void ngx_linux_sendfile_thread_handler(void *data, ngx_log_t *log); +#endif + + /* * On Linux up to 2.4.21 sendfile() (syscall #187) works with 32-bit * offsets only, and the including <sys/sendfile.h> breaks the compiling, @@ -31,20 +47,18 @@ ngx_linux_sendfile_chain(ngx_connection_t *c, ngx_chain_t *in, off_t limit) { int tcp_nodelay; - off_t send, prev_send, sent; - size_t file_size; + off_t send, prev_send; + size_t file_size, sent; ssize_t n; ngx_err_t err; ngx_buf_t *file; - ngx_uint_t eintr; ngx_event_t *wev; ngx_chain_t *cl; ngx_iovec_t header; struct iovec headers[NGX_IOVS_PREALLOCATE]; -#if (NGX_HAVE_SENDFILE64) - off_t offset; -#else - int32_t offset; +#if (NGX_THREADS) + ngx_int_t rc; + ngx_uint_t thread_handled, thread_complete; #endif wev = c->write; @@ -67,8 +81,11 @@ header.nalloc = NGX_IOVS_PREALLOCATE; for ( ;; ) { - eintr = 0; prev_send = send; +#if (NGX_THREADS) + thread_handled = 0; + thread_complete = 0; +#endif /* create the iovec and coalesce the neighbouring bufs */ @@ -161,43 +178,38 @@ return NGX_CHAIN_ERROR; } #endif -#if (NGX_HAVE_SENDFILE64) - offset = file->file_pos; -#else - offset = (int32_t) file->file_pos; -#endif - - ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, - "sendfile: @%O %uz", file->file_pos, file_size); - - n = sendfile(c->fd, file->file->fd, &offset, file_size); - if (n == -1) { - err = ngx_errno; +#if (NGX_THREADS) + if (file->file->thread_handler) { + rc = ngx_linux_sendfile_thread(c, file, file_size, &sent); + + switch (rc) { + case NGX_OK: + thread_handled = 1; + break; - switch (err) { - case NGX_EAGAIN: + case NGX_DONE: + thread_complete = 1; break; - case NGX_EINTR: - eintr = 1; + case NGX_AGAIN: break; - default: - wev->error = 1; - ngx_connection_error(c, err, "sendfile() failed"); + default: /* NGX_ERROR */ return NGX_CHAIN_ERROR; } - ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err, - "sendfile() is not ready"); - } + } else +#endif + { + n = ngx_linux_sendfile(c, file, file_size); - sent = n > 0 ? n : 0; + if (n == NGX_ERROR) { + return NGX_CHAIN_ERROR; + } - ngx_log_debug4(NGX_LOG_DEBUG_EVENT, c->log, 0, - "sendfile: %z, @%O %O:%uz", - n, file->file_pos, sent, file_size); + sent = (n == NGX_AGAIN) ? 0 : n; + } } else { n = ngx_writev(c, &header); @@ -213,12 +225,17 @@ in = ngx_chain_update_sent(in, sent); - if (eintr) { - send = prev_send; - continue; - } + if ((size_t) (send - prev_send) != sent) { +#if (NGX_THREADS) + if (thread_handled) { + return in; + } - if (send - prev_send != sent) { + if (thread_complete) { + send = prev_send + sent; + continue; + } +#endif wev->ready = 0; return in; } @@ -228,3 +245,175 @@ } } } + + +static ssize_t +ngx_linux_sendfile(ngx_connection_t *c, ngx_buf_t *file, size_t size) +{ +#if (NGX_HAVE_SENDFILE64) + off_t offset; +#else + int32_t offset; +#endif + ssize_t n; + ngx_err_t err; + +#if (NGX_HAVE_SENDFILE64) + offset = file->file_pos; +#else + offset = (int32_t) file->file_pos; +#endif + +eintr: + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, c->log, 0, + "sendfile: @%O %uz", file->file_pos, size); + + n = sendfile(c->fd, file->file->fd, &offset, size); + + if (n == -1) { + err = ngx_errno; + + switch (err) { + case NGX_EAGAIN: + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err, + "sendfile() is not ready"); + return NGX_AGAIN; + + case NGX_EINTR: + ngx_log_debug0(NGX_LOG_DEBUG_EVENT, c->log, err, + "sendfile() was interrupted"); + goto eintr; + + default: + c->write->error = 1; + ngx_connection_error(c, err, "sendfile() failed"); + return NGX_ERROR; + } + } + + ngx_log_debug3(NGX_LOG_DEBUG_EVENT, c->log, 0, "sendfile: %z of %uz @%O", + n, size, file->file_pos); + + return n; +} + + +#if (NGX_THREADS) + +typedef struct { + ngx_buf_t *file; + ngx_socket_t socket; + size_t size; + + size_t sent; + ngx_err_t err; +} ngx_linux_sendfile_ctx_t; + + +static ngx_int_t +ngx_linux_sendfile_thread(ngx_connection_t *c, ngx_buf_t *file, size_t size, + size_t *sent) +{ + ngx_uint_t flags; + ngx_event_t *wev; + ngx_thread_task_t *task; + ngx_linux_sendfile_ctx_t *ctx; + + ngx_log_debug3(NGX_LOG_DEBUG_CORE, c->log, 0, + "linux sendfile thread: %d, %uz, %O", + file->file->fd, size, file->file_pos); + + task = c->sendfile_task; + + if (task == NULL) { + task = ngx_thread_task_alloc(c->pool, sizeof(ngx_linux_sendfile_ctx_t)); + if (task == NULL) { + return NGX_ERROR; + } + + task->handler = ngx_linux_sendfile_thread_handler; + + c->sendfile_task = task; + } + + ctx = task->ctx; + wev = c->write; + + if (task->event.complete) { + task->event.complete = 0; + + if (ctx->err && ctx->err != NGX_EAGAIN) { + wev->error = 1; + ngx_connection_error(c, ctx->err, "sendfile() failed"); + return NGX_ERROR; + } + + *sent = ctx->sent; + + return (ctx->sent == ctx->size) ? NGX_DONE : NGX_AGAIN; + } + + ctx->file = file; + ctx->socket = c->fd; + ctx->size = size; + + if (wev->active) { + flags = (ngx_event_flags & NGX_USE_CLEAR_EVENT) ? NGX_CLEAR_EVENT + : NGX_LEVEL_EVENT; + + if (ngx_del_event(wev, NGX_WRITE_EVENT, flags) == NGX_ERROR) { + return NGX_ERROR; + } + } + + if (file->file->thread_handler(task, file->file) != NGX_OK) { + return NGX_ERROR; + } + + *sent = 0; + + return NGX_OK; +} + + +static void +ngx_linux_sendfile_thread_handler(void *data, ngx_log_t *log) +{ + ngx_linux_sendfile_ctx_t *ctx = data; + + off_t offset; + ssize_t n; + ngx_buf_t *file; + + ngx_log_debug0(NGX_LOG_DEBUG_CORE, log, 0, "linux sendfile thread handler"); + + file = ctx->file; + offset = file->file_pos; + +again: + + n = sendfile(ctx->socket, file->file->fd, &offset, ctx->size); + + if (n == -1) { + ctx->err = ngx_errno; + + } else { + ctx->sent = n; + ctx->err = 0; + } + +#if 0 + ngx_time_update(); +#endif + + ngx_log_debug4(NGX_LOG_DEBUG_EVENT, log, 0, + "sendfile: %z (err: %i) of %uz @%O", + n, ctx->err, ctx->size, file->file_pos); + + if (ctx->err == NGX_EINTR) { + goto again; + } +} + +#endif /* NGX_THREADS */

@@ -23,10 +23,6 @@ static void ngx_worker_process_init(ngx_cycle_t *cycle, ngx_int_t worker); static void ngx_worker_process_exit(ngx_cycle_t *cycle); static void ngx_channel_handler(ngx_event_t *ev); -#if (NGX_THREADS) -static void ngx_wakeup_worker_threads(ngx_cycle_t *cycle); -static ngx_thread_value_t ngx_worker_thread_cycle(void *data); -#endif static void ngx_cache_manager_process_cycle(ngx_cycle_t *cycle, void *data); static void ngx_cache_manager_process_handler(ngx_event_t *ev); static void ngx_cache_loader_process_handler(ngx_event_t *ev); @@ -34,7 +30,6 @@ ngx_uint_t ngx_process; ngx_pid_t ngx_pid; -ngx_uint_t ngx_threaded; sig_atomic_t ngx_reap; sig_atomic_t ngx_sigio; @@ -56,12 +51,6 @@ ngx_uint_t ngx_restart; -#if (NGX_THREADS) -volatile ngx_thread_t ngx_threads[NGX_MAX_THREADS]; -ngx_int_t ngx_threads_n; -#endif - - static u_char master_process[] = "master process"; @@ -747,52 +736,6 @@ ngx_setproctitle("worker process"); -#if (NGX_THREADS) - { - ngx_int_t n; - ngx_err_t err; - ngx_core_conf_t *ccf; - - ccf = (ngx_core_conf_t *) ngx_get_conf(cycle->conf_ctx, ngx_core_module); - - if (ngx_threads_n) { - if (ngx_init_threads(ngx_threads_n, ccf->thread_stack_size, cycle) - == NGX_ERROR) - { - /* fatal */ - exit(2); - } - - err = ngx_thread_key_create(&ngx_core_tls_key); - if (err != 0) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, err, - ngx_thread_key_create_n " failed"); - /* fatal */ - exit(2); - } - - for (n = 0; n < ngx_threads_n; n++) { - - ngx_threads[n].cv = ngx_cond_init(cycle->log); - - if (ngx_threads[n].cv == NULL) { - /* fatal */ - exit(2); - } - - if (ngx_create_thread((ngx_tid_t *) &ngx_threads[n].tid, - ngx_worker_thread_cycle, - (void *) &ngx_threads[n], cycle->log) - != 0) - { - /* fatal */ - exit(2); - } - } - } - } -#endif - for ( ;; ) { if (ngx_exiting) { @@ -1032,12 +975,6 @@ ngx_uint_t i; ngx_connection_t *c; -#if (NGX_THREADS) - ngx_terminate = 1; - - ngx_wakeup_worker_threads(cycle); -#endif - for (i = 0; ngx_modules[i]; i++) { if (ngx_modules[i]->exit_process) { ngx_modules[i]->exit_process(cycle); @@ -1181,132 +1118,6 @@ } -#if (NGX_THREADS) - -static void -ngx_wakeup_worker_threads(ngx_cycle_t *cycle) -{ - ngx_int_t i; - ngx_uint_t live; - - for ( ;; ) { - - live = 0; - - for (i = 0; i < ngx_threads_n; i++) { - if (ngx_threads[i].state < NGX_THREAD_EXIT) { - if (ngx_cond_signal(ngx_threads[i].cv) == NGX_ERROR) { - ngx_threads[i].state = NGX_THREAD_DONE; - - } else { - live = 1; - } - } - - if (ngx_threads[i].state == NGX_THREAD_EXIT) { - ngx_thread_join(ngx_threads[i].tid, NULL); - ngx_threads[i].state = NGX_THREAD_DONE; - } - } - - if (live == 0) { - ngx_log_debug0(NGX_LOG_DEBUG_CORE, cycle->log, 0, - "all worker threads are joined"); - - /* STUB */ - ngx_done_events(cycle); - - return; - } - - ngx_sched_yield(); - } -} - - -static ngx_thread_value_t -ngx_worker_thread_cycle(void *data) -{ - ngx_thread_t *thr = data; - - sigset_t set; - ngx_err_t err; - ngx_core_tls_t *tls; - ngx_cycle_t *cycle; - - cycle = (ngx_cycle_t *) ngx_cycle; - - sigemptyset(&set); - sigaddset(&set, ngx_signal_value(NGX_RECONFIGURE_SIGNAL)); - sigaddset(&set, ngx_signal_value(NGX_REOPEN_SIGNAL)); - sigaddset(&set, ngx_signal_value(NGX_CHANGEBIN_SIGNAL)); - - err = ngx_thread_sigmask(SIG_BLOCK, &set, NULL); - if (err) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, err, - ngx_thread_sigmask_n " failed"); - return (ngx_thread_value_t) 1; - } - - ngx_log_debug1(NGX_LOG_DEBUG_CORE, cycle->log, 0, - "thread " NGX_TID_T_FMT " started", ngx_thread_self()); - - ngx_setthrtitle("worker thread"); - - tls = ngx_calloc(sizeof(ngx_core_tls_t), cycle->log); - if (tls == NULL) { - return (ngx_thread_value_t) 1; - } - - err = ngx_thread_set_tls(ngx_core_tls_key, tls); - if (err != 0) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, err, - ngx_thread_set_tls_n " failed"); - return (ngx_thread_value_t) 1; - } - - for ( ;; ) { - thr->state = NGX_THREAD_FREE; - -#if 0 - if (ngx_cond_wait(thr->cv, ngx_posted_events_mutex) == NGX_ERROR) { - return (ngx_thread_value_t) 1; - } -#endif - - if (ngx_terminate) { - thr->state = NGX_THREAD_EXIT; - - ngx_log_debug1(NGX_LOG_DEBUG_CORE, cycle->log, 0, - "thread " NGX_TID_T_FMT " is done", - ngx_thread_self()); - - return (ngx_thread_value_t) 0; - } - - thr->state = NGX_THREAD_BUSY; - -#if 0 - if (ngx_event_thread_process_posted(cycle) == NGX_ERROR) { - return (ngx_thread_value_t) 1; - } - - if (ngx_event_thread_process_posted(cycle) == NGX_ERROR) { - return (ngx_thread_value_t) 1; - } -#endif - - if (ngx_process_changes) { - if (ngx_process_changes(cycle, 1) == NGX_ERROR) { - return (ngx_thread_value_t) 1; - } - } - } -} - -#endif - - static void ngx_cache_manager_process_cycle(ngx_cycle_t *cycle, void *data) {

@@ -43,7 +43,6 @@ extern ngx_pid_t ngx_new_binary; extern ngx_uint_t ngx_inherited; extern ngx_uint_t ngx_daemonized; -extern ngx_uint_t ngx_threaded; extern ngx_uint_t ngx_exiting; extern sig_atomic_t ngx_reap;

@@ -14,114 +14,57 @@ #if (NGX_THREADS) -#define NGX_MAX_THREADS 128 - -#if (NGX_USE_RFORK) -#include <ngx_freebsd_rfork_thread.h> - - -#else /* use pthreads */ - #include <pthread.h> -typedef pthread_t ngx_tid_t; - -#define ngx_thread_self() pthread_self() -#define ngx_log_tid (int) ngx_thread_self() -#if (NGX_FREEBSD) && !(NGX_LINUXTHREADS) -#define NGX_TID_T_FMT "%p" -#else -#define NGX_TID_T_FMT "%d" -#endif +typedef pthread_mutex_t ngx_thread_mutex_t; +ngx_int_t ngx_thread_mutex_create(ngx_thread_mutex_t *mtx, ngx_log_t *log); +ngx_int_t ngx_thread_mutex_destroy(ngx_thread_mutex_t *mtx, ngx_log_t *log); +ngx_int_t ngx_thread_mutex_lock(ngx_thread_mutex_t *mtx, ngx_log_t *log); +ngx_int_t ngx_thread_mutex_unlock(ngx_thread_mutex_t *mtx, ngx_log_t *log); -typedef pthread_key_t ngx_tls_key_t; -#define ngx_thread_key_create(key) pthread_key_create(key, NULL) -#define ngx_thread_key_create_n "pthread_key_create()" -#define ngx_thread_set_tls pthread_setspecific -#define ngx_thread_set_tls_n "pthread_setspecific()" -#define ngx_thread_get_tls pthread_getspecific +typedef pthread_cond_t ngx_thread_cond_t; +ngx_int_t ngx_thread_cond_create(ngx_thread_cond_t *cond, ngx_log_t *log); +ngx_int_t ngx_thread_cond_destroy(ngx_thread_cond_t *cond, ngx_log_t *log); +ngx_int_t ngx_thread_cond_signal(ngx_thread_cond_t *cond, ngx_log_t *log); +ngx_int_t ngx_thread_cond_wait(ngx_thread_cond_t *cond, ngx_thread_mutex_t *mtx, + ngx_log_t *log); -#define NGX_MUTEX_LIGHT 0 -typedef struct { - pthread_mutex_t mutex; - ngx_log_t *log; -} ngx_mutex_t; +#if (NGX_LINUX) -typedef struct { - pthread_cond_t cond; - ngx_log_t *log; -} ngx_cond_t; +typedef pid_t ngx_tid_t; +#define NGX_TID_T_FMT "%P" -#define ngx_thread_sigmask pthread_sigmask -#define ngx_thread_sigmask_n "pthread_sigmask()" +#elif (NGX_FREEBSD) -#define ngx_thread_join(t, p) pthread_join(t, p) +typedef uint32_t ngx_tid_t; +#define NGX_TID_T_FMT "%uD" -#define ngx_setthrtitle(n) +#elif (NGX_DARWIN) +typedef uint64_t ngx_tid_t; +#define NGX_TID_T_FMT "%uA" +#else -ngx_int_t ngx_mutex_trylock(ngx_mutex_t *m); -void ngx_mutex_lock(ngx_mutex_t *m); -void ngx_mutex_unlock(ngx_mutex_t *m); +typedef uint64_t ngx_tid_t; +#define NGX_TID_T_FMT "%uA" #endif +ngx_tid_t ngx_thread_tid(void); -#define ngx_thread_volatile volatile - - -typedef struct { - ngx_tid_t tid; - ngx_cond_t *cv; - ngx_uint_t state; -} ngx_thread_t; - -#define NGX_THREAD_FREE 1 -#define NGX_THREAD_BUSY 2 -#define NGX_THREAD_EXIT 3 -#define NGX_THREAD_DONE 4 - -extern ngx_int_t ngx_threads_n; -extern volatile ngx_thread_t ngx_threads[NGX_MAX_THREADS]; +#define ngx_log_tid ngx_thread_tid() - -typedef void * ngx_thread_value_t; - -ngx_int_t ngx_init_threads(int n, size_t size, ngx_cycle_t *cycle); -ngx_err_t ngx_create_thread(ngx_tid_t *tid, - ngx_thread_value_t (*func)(void *arg), void *arg, ngx_log_t *log); - -ngx_mutex_t *ngx_mutex_init(ngx_log_t *log, ngx_uint_t flags); -void ngx_mutex_destroy(ngx_mutex_t *m); - - -ngx_cond_t *ngx_cond_init(ngx_log_t *log); -void ngx_cond_destroy(ngx_cond_t *cv); -ngx_int_t ngx_cond_wait(ngx_cond_t *cv, ngx_mutex_t *m); -ngx_int_t ngx_cond_signal(ngx_cond_t *cv); - - -#else /* !NGX_THREADS */ - -#define ngx_thread_volatile +#else #define ngx_log_tid 0 #define NGX_TID_T_FMT "%d" -#define ngx_mutex_trylock(m) NGX_OK -#define ngx_mutex_lock(m) -#define ngx_mutex_unlock(m) - -#define ngx_cond_signal(cv) - -#define ngx_thread_main() 1 - #endif

@@ -0,0 +1,87 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) Nginx, Inc. + */ + + +#include <ngx_config.h> +#include <ngx_core.h> + + +ngx_int_t +ngx_thread_cond_create(ngx_thread_cond_t *cond, ngx_log_t *log) +{ + ngx_err_t err; + + err = pthread_cond_init(cond, NULL); + if (err == 0) { + ngx_log_debug1(NGX_LOG_DEBUG_CORE, log, 0, + "pthread_cond_init(%p)", cond); + return NGX_OK; + } + + ngx_log_error(NGX_LOG_EMERG, log, err, "pthread_cond_init() failed"); + return NGX_ERROR; +} + + +ngx_int_t +ngx_thread_cond_destroy(ngx_thread_cond_t *cond, ngx_log_t *log) +{ + ngx_err_t err; + + err = pthread_cond_destroy(cond); + if (err == 0) { + ngx_log_debug1(NGX_LOG_DEBUG_CORE, log, 0, + "pthread_cond_destroy(%p)", cond); + return NGX_OK; + } + + ngx_log_error(NGX_LOG_EMERG, log, err, "pthread_cond_destroy() failed"); + return NGX_ERROR; +} + + +ngx_int_t +ngx_thread_cond_signal(ngx_thread_cond_t *cond, ngx_log_t *log) +{ + ngx_err_t err; + + err = pthread_cond_signal(cond); + if (err == 0) { + ngx_log_debug1(NGX_LOG_DEBUG_CORE, log, 0, + "pthread_cond_signal(%p)", cond); + return NGX_OK; + } + + ngx_log_error(NGX_LOG_EMERG, log, err, "pthread_cond_signal() failed"); + return NGX_ERROR; +} + + +ngx_int_t +ngx_thread_cond_wait(ngx_thread_cond_t *cond, ngx_thread_mutex_t *mtx, + ngx_log_t *log) +{ + ngx_err_t err; + + ngx_log_debug1(NGX_LOG_DEBUG_CORE, log, 0, + "pthread_cond_wait(%p) enter", cond); + + err = pthread_cond_wait(cond, mtx); + +#if 0 + ngx_time_update(); +#endif + + if (err == 0) { + ngx_log_debug1(NGX_LOG_DEBUG_CORE, log, 0, + "pthread_cond_wait(%p) exit", cond); + return NGX_OK; + } + + ngx_log_error(NGX_LOG_ALERT, log, err, "pthread_cond_wait() failed"); + + return NGX_ERROR; +}

@@ -0,0 +1,70 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) Nginx, Inc. + */ + + +#include <ngx_config.h> +#include <ngx_core.h> +#include <ngx_thread_pool.h> + + +#if (NGX_LINUX) + +/* + * Linux thread id is a pid of thread created by clone(2), + * glibc does not provide a wrapper for gettid(). + */ + +ngx_tid_t +ngx_thread_tid(void) +{ + return syscall(SYS_gettid); +} + +#elif (NGX_FREEBSD) && (__FreeBSD_version >= 900031) + +#include <pthread_np.h> + +ngx_tid_t +ngx_thread_tid(void) +{ + return pthread_getthreadid_np(); +} + +#elif (NGX_DARWIN) + +/* + * MacOSX thread has two thread ids: + * + * 1) MacOSX 10.6 (Snow Leoprad) has pthread_threadid_np() returning + * an uint64_t value, which is obtained using the __thread_selfid() + * syscall. It is a number above 300,000. + */ + +ngx_tid_t +ngx_thread_tid(void) +{ + uint64_t tid; + + (void) pthread_threadid_np(NULL, &tid); + return tid; +} + +/* + * 2) Kernel thread mach_port_t returned by pthread_mach_thread_np(). + * It is a number in range 100-100,000. + * + * return pthread_mach_thread_np(pthread_self()); + */ + +#else + +ngx_tid_t +ngx_thread_tid(void) +{ + return (uint64_t) (uintptr_t) pthread_self(); +} + +#endif

@@ -0,0 +1,174 @@ + +/* + * Copyright (C) Igor Sysoev + * Copyright (C) Nginx, Inc. + */ + +#include <ngx_config.h> +#include <ngx_core.h> + + +/* + * All modern pthread mutex implementations try to acquire a lock + * atomically in userland before going to sleep in kernel. Some + * spins before the sleeping. + * + * In Solaris since version 8 all mutex types spin before sleeping. + * The default spin count is 1000. It can be overridden using + * _THREAD_ADAPTIVE_SPIN=100 environment variable. + * + * In MacOSX all mutex types spin to acquire a lock protecting a mutex's + * internals. If the mutex is busy, thread calls Mach semaphore_wait(). + * + * + * PTHREAD_MUTEX_NORMAL lacks deadlock detection and is the fastest + * mutex type. + * + * Linux: No spinning. The internal name PTHREAD_MUTEX_TIMED_NP + * remains from the times when pthread_mutex_timedlock() was + * non-standard extension. Alias name: PTHREAD_MUTEX_FAST_NP. + * FreeBSD: No spinning. + * + * + * PTHREAD_MUTEX_ERRORCHECK is usually as fast as PTHREAD_MUTEX_NORMAL + * yet has lightweight deadlock detection. + * + * Linux: No spinning. The internal name: PTHREAD_MUTEX_ERRORCHECK_NP. + * FreeBSD: No spinning. + * + * + * PTHREAD_MUTEX_RECURSIVE allows recursive locking. + * + * Linux: No spinning. The internal name: PTHREAD_MUTEX_RECURSIVE_NP. + * FreeBSD: No spinning. + * + * + * PTHREAD_MUTEX_ADAPTIVE_NP spins on SMP systems before sleeping. + * + * Linux: No deadlock detection. Dynamically changes a spin count + * for each mutex from 10 to 100 based on spin count taken + * previously. + * FreeBSD: Deadlock detection. The default spin count is 2000. + * It can be overriden using LIBPTHREAD_SPINLOOPS environment + * variable or by pthread_mutex_setspinloops_np(). If a lock + * is still busy, sched_yield() can be called on both UP and + * SMP systems. The default yield loop count is zero, but + * it can be set by LIBPTHREAD_YIELDLOOPS environment + * variable or by pthread_mutex_setyieldloops_np(). + * Solaris: No PTHREAD_MUTEX_ADAPTIVE_NP. + * MacOSX: No PTHREAD_MUTEX_ADAPTIVE_NP. + * + * + * PTHREAD_MUTEX_ELISION_NP is a Linux extension to elide locks using + * Intel Restricted Transactional Memory. It is the most suitable for + * rwlock pattern access because it allows simultaneous reads without lock. + * Supported since glibc 2.18. + * + * + * PTHREAD_MUTEX_DEFAULT is default mutex type. + * + * Linux: PTHREAD_MUTEX_NORMAL. + * FreeBSD: PTHREAD_MUTEX_ERRORCHECK. + * Solaris: PTHREAD_MUTEX_NORMAL. + * MacOSX: PTHREAD_MUTEX_NORMAL. + */ + + +ngx_int_t +ngx_thread_mutex_create(ngx_thread_mutex_t *mtx, ngx_log_t *log) +{ + ngx_err_t err; + pthread_mutexattr_t attr; + + err = pthread_mutexattr_init(&attr); + if (err != 0) { + ngx_log_error(NGX_LOG_EMERG, log, err, + "pthread_mutexattr_init() failed"); + return NGX_ERROR; + } + + err = pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); + if (err != 0) { + ngx_log_error(NGX_LOG_EMERG, log, err, + "pthread_mutexattr_settype" + "(PTHREAD_MUTEX_ERRORCHECK) failed"); + return NGX_ERROR; + } + + err = pthread_mutex_init(mtx, &attr); + if (err != 0) { + ngx_log_error(NGX_LOG_EMERG, log, err, + "pthread_mutex_init() failed"); + return NGX_ERROR; + } + + err = pthread_mutexattr_destroy(&attr); + if (err != 0) { + ngx_log_error(NGX_LOG_ALERT, log, err, + "pthread_mutexattr_destroy() failed"); + } + + ngx_log_debug1(NGX_LOG_DEBUG_CORE, log, 0, + "pthread_mutex_init(%p)", mtx); + return NGX_OK; +} + + +ngx_int_t +ngx_thread_mutex_destroy(ngx_thread_mutex_t *mtx, ngx_log_t *log) +{ + ngx_err_t err; + + err = pthread_mutex_destroy(mtx); + if (err != 0) { + ngx_log_error(NGX_LOG_ALERT, log, err, + "pthread_mutex_destroy() failed"); + return NGX_ERROR; + } + + ngx_log_debug1(NGX_LOG_DEBUG_CORE, log, 0, + "pthread_mutex_destroy(%p)", mtx); + return NGX_OK; +} + + +ngx_int_t +ngx_thread_mutex_lock(ngx_thread_mutex_t *mtx, ngx_log_t *log) +{ + ngx_err_t err; + + ngx_log_debug1(NGX_LOG_DEBUG_CORE, log, 0, + "pthread_mutex_lock(%p) enter", mtx); + + err = pthread_mutex_lock(mtx); + if (err == 0) { + return NGX_OK; + } + + ngx_log_error(NGX_LOG_ALERT, log, err, "pthread_mutex_lock() failed"); + + return NGX_ERROR; +} + + +ngx_int_t +ngx_thread_mutex_unlock(ngx_thread_mutex_t *mtx, ngx_log_t *log) +{ + ngx_err_t err; + + err = pthread_mutex_unlock(mtx); + +#if 0 + ngx_time_update(); +#endif + + if (err == 0) { + ngx_log_debug1(NGX_LOG_DEBUG_CORE, log, 0, + "pthread_mutex_unlock(%p) exit", mtx); + return NGX_OK; + } + + ngx_log_error(NGX_LOG_ALERT, log, err, "pthread_mutex_unlock() failed"); + + return NGX_ERROR; +}

@@ -31,8 +31,10 @@ struct crypt_data cd; cd.initialized = 0; +#ifdef __GLIBC__ /* work around the glibc bug */ cd.current_salt[0] = ~salt[0]; +#endif value = crypt_r((char *) key, (char *) salt, &cd); @@ -62,16 +64,6 @@ size_t len; ngx_err_t err; -#if (NGX_THREADS && NGX_NONREENTRANT_CRYPT) - - /* crypt() is a time consuming function, so we only try to lock */ - - if (ngx_mutex_trylock(ngx_crypt_mutex) != NGX_OK) { - return NGX_AGAIN; - } - -#endif - value = crypt((char *) key, (char *) salt); if (value) { @@ -79,25 +71,15 @@ *encrypted = ngx_pnalloc(pool, len); if (*encrypted == NULL) { -#if (NGX_THREADS && NGX_NONREENTRANT_CRYPT) - ngx_mutex_unlock(ngx_crypt_mutex); -#endif return NGX_ERROR; } ngx_memcpy(*encrypted, value, len); -#if (NGX_THREADS && NGX_NONREENTRANT_CRYPT) - ngx_mutex_unlock(ngx_crypt_mutex); -#endif return NGX_OK; } err = ngx_errno; -#if (NGX_THREADS && NGX_NONREENTRANT_CRYPT) - ngx_mutex_unlock(ngx_crypt_mutex); -#endif - ngx_log_error(NGX_LOG_CRIT, pool->log, err, "crypt() failed"); return NGX_ERROR;

@@ -1,286 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) Nginx, Inc. - */ - - -#include <ngx_config.h> -#include <ngx_core.h> -#include <ngx_event.h> - - -static ngx_int_t ngx_event_busy_lock_look_cacheable(ngx_event_busy_lock_t *bl, - ngx_event_busy_lock_ctx_t *ctx); -static void ngx_event_busy_lock_handler(ngx_event_t *ev); -static void ngx_event_busy_lock_posted_handler(ngx_event_t *ev); - - -/* - * NGX_OK: the busy lock is held - * NGX_AGAIN: the all busy locks are held but we will wait the specified time - * NGX_BUSY: ctx->timer == 0: there are many the busy locks - * ctx->timer != 0: there are many the waiting locks - */ - -ngx_int_t -ngx_event_busy_lock(ngx_event_busy_lock_t *bl, ngx_event_busy_lock_ctx_t *ctx) -{ - ngx_int_t rc; - - ngx_mutex_lock(bl->mutex); - - ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ctx->event->log, 0, - "event busy lock: b:%d mb:%d", - bl->busy, bl->max_busy); - - if (bl->busy < bl->max_busy) { - bl->busy++; - - rc = NGX_OK; - - } else if (ctx->timer && bl->waiting < bl->max_waiting) { - bl->waiting++; - ngx_add_timer(ctx->event, ctx->timer); - ctx->event->handler = ngx_event_busy_lock_handler; - - if (bl->events) { - bl->last->next = ctx; - - } else { - bl->events = ctx; - } - - bl->last = ctx; - - rc = NGX_AGAIN; - - } else { - rc = NGX_BUSY; - } - - ngx_mutex_unlock(bl->mutex); - - return rc; -} - - -ngx_int_t -ngx_event_busy_lock_cacheable(ngx_event_busy_lock_t *bl, - ngx_event_busy_lock_ctx_t *ctx) -{ - ngx_int_t rc; - - ngx_mutex_lock(bl->mutex); - - rc = ngx_event_busy_lock_look_cacheable(bl, ctx); - - ngx_log_debug3(NGX_LOG_DEBUG_EVENT, ctx->event->log, 0, - "event busy lock: %d w:%d mw:%d", - rc, bl->waiting, bl->max_waiting); - - /* - * NGX_OK: no the same request, there is free slot and we locked it - * NGX_BUSY: no the same request and there is no free slot - * NGX_AGAIN: the same request is processing - */ - - if (rc == NGX_AGAIN) { - - if (ctx->timer && bl->waiting < bl->max_waiting) { - bl->waiting++; - ngx_add_timer(ctx->event, ctx->timer); - ctx->event->handler = ngx_event_busy_lock_handler; - - if (bl->events == NULL) { - bl->events = ctx; - } else { - bl->last->next = ctx; - } - bl->last = ctx; - - } else { - rc = NGX_BUSY; - } - } - - ngx_mutex_unlock(bl->mutex); - - return rc; -} - - -void -ngx_event_busy_unlock(ngx_event_busy_lock_t *bl, - ngx_event_busy_lock_ctx_t *ctx) -{ - ngx_event_t *ev; - ngx_event_busy_lock_ctx_t *wakeup; - - ngx_mutex_lock(bl->mutex); - - if (bl->events) { - wakeup = bl->events; - bl->events = bl->events->next; - - } else { - wakeup = NULL; - bl->busy--; - } - - /* - * MP: all ctx's and their queue must be in shared memory, - * each ctx has pid to wake up - */ - - if (wakeup == NULL) { - ngx_mutex_unlock(bl->mutex); - return; - } - - if (ctx->md5) { - for (wakeup = bl->events; wakeup; wakeup = wakeup->next) { - if (wakeup->md5 == NULL || wakeup->slot != ctx->slot) { - continue; - } - - wakeup->handler = ngx_event_busy_lock_posted_handler; - wakeup->cache_updated = 1; - - ev = wakeup->event; - - ngx_post_event(ev, &ngx_posted_events); - } - - ngx_mutex_unlock(bl->mutex); - - } else { - bl->waiting--; - - ngx_mutex_unlock(bl->mutex); - - wakeup->handler = ngx_event_busy_lock_posted_handler; - wakeup->locked = 1; - - ev = wakeup->event; - - if (ev->timer_set) { - ngx_del_timer(ev); - } - - ngx_post_event(ev, &ngx_posted_events); - } -} - - -void -ngx_event_busy_lock_cancel(ngx_event_busy_lock_t *bl, - ngx_event_busy_lock_ctx_t *ctx) -{ - ngx_event_busy_lock_ctx_t *c, *p; - - ngx_mutex_lock(bl->mutex); - - bl->waiting--; - - if (ctx == bl->events) { - bl->events = ctx->next; - - } else { - p = bl->events; - for (c = bl->events->next; c; c = c->next) { - if (c == ctx) { - p->next = ctx->next; - break; - } - p = c; - } - } - - ngx_mutex_unlock(bl->mutex); -} - - -static ngx_int_t -ngx_event_busy_lock_look_cacheable(ngx_event_busy_lock_t *bl, - ngx_event_busy_lock_ctx_t *ctx) -{ - ngx_int_t free; - ngx_uint_t i, bit, cacheable, mask; - - bit = 0; - cacheable = 0; - free = -1; - -#if (NGX_SUPPRESS_WARN) - mask = 0; -#endif - - for (i = 0; i < bl->max_busy; i++) { - - if ((bit & 7) == 0) { - mask = bl->md5_mask[i / 8]; - } - - if (mask & 1) { - if (ngx_memcmp(&bl->md5[i * 16], ctx->md5, 16) == 0) { - ctx->waiting = 1; - ctx->slot = i; - return NGX_AGAIN; - } - cacheable++; - - } else if (free == -1) { - free = i; - } - - if (cacheable == bl->cacheable) { - if (free == -1 && cacheable < bl->max_busy) { - free = i + 1; - } - - break; - } - - mask >>= 1; - bit++; - } - - if (free == -1) { - return NGX_BUSY; - } - -#if 0 - if (bl->busy == bl->max_busy) { - return NGX_BUSY; - } -#endif - - ngx_memcpy(&bl->md5[free * 16], ctx->md5, 16); - bl->md5_mask[free / 8] |= 1 << (free & 7); - ctx->slot = free; - - bl->cacheable++; - bl->busy++; - - return NGX_OK; -} - - -static void -ngx_event_busy_lock_handler(ngx_event_t *ev) -{ - ev->handler = ngx_event_busy_lock_posted_handler; - - ngx_post_event(ev, &ngx_posted_events); -} - - -static void -ngx_event_busy_lock_posted_handler(ngx_event_t *ev) -{ - ngx_event_busy_lock_ctx_t *ctx; - - ctx = ev->data; - ctx->handler(ev); -}

@@ -1,65 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) Nginx, Inc. - */ - - -#ifndef _NGX_EVENT_BUSY_LOCK_H_INCLUDED_ -#define _NGX_EVENT_BUSY_LOCK_H_INCLUDED_ - - -#include <ngx_config.h> -#include <ngx_core.h> -#include <ngx_event.h> - -typedef struct ngx_event_busy_lock_ctx_s ngx_event_busy_lock_ctx_t; - -struct ngx_event_busy_lock_ctx_s { - ngx_event_t *event; - ngx_event_handler_pt handler; - void *data; - ngx_msec_t timer; - - unsigned locked:1; - unsigned waiting:1; - unsigned cache_updated:1; - - char *md5; - ngx_int_t slot; - - ngx_event_busy_lock_ctx_t *next; -}; - - -typedef struct { - u_char *md5_mask; - char *md5; - ngx_uint_t cacheable; - - ngx_uint_t busy; - ngx_uint_t max_busy; - - ngx_uint_t waiting; - ngx_uint_t max_waiting; - - ngx_event_busy_lock_ctx_t *events; - ngx_event_busy_lock_ctx_t *last; - -#if (NGX_THREADS) - ngx_mutex_t *mutex; -#endif -} ngx_event_busy_lock_t; - - -ngx_int_t ngx_event_busy_lock(ngx_event_busy_lock_t *bl, - ngx_event_busy_lock_ctx_t *ctx); -ngx_int_t ngx_event_busy_lock_cacheable(ngx_event_busy_lock_t *bl, - ngx_event_busy_lock_ctx_t *ctx); -void ngx_event_busy_unlock(ngx_event_busy_lock_t *bl, - ngx_event_busy_lock_ctx_t *ctx); -void ngx_event_busy_lock_cancel(ngx_event_busy_lock_t *bl, - ngx_event_busy_lock_ctx_t *ctx); - - -#endif /* _NGX_EVENT_BUSY_LOCK_H_INCLUDED_ */

@@ -1,70 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) Nginx, Inc. - */ - - -#include <ngx_config.h> -#include <ngx_core.h> -#include <ngx_event.h> - - -ngx_int_t ngx_event_mutex_timedlock(ngx_event_mutex_t *m, ngx_msec_t timer, - ngx_event_t *ev) -{ - ngx_log_debug2(NGX_LOG_DEBUG_EVENT, ev->log, 0, - "lock event mutex %p lock:%XD", m, m->lock); - - if (m->lock) { - - if (m->events == NULL) { - m->events = ev; - - } else { - m->last->next = ev; - } - - m->last = ev; - ev->next = NULL; - -#if (NGX_THREADS0) - ev->light = 1; -#endif - - ngx_add_timer(ev, timer); - - return NGX_AGAIN; - } - - m->lock = 1; - - return NGX_OK; -} - - -ngx_int_t ngx_event_mutex_unlock(ngx_event_mutex_t *m, ngx_log_t *log) -{ - ngx_event_t *ev; - - if (m->lock == 0) { - ngx_log_error(NGX_LOG_ALERT, log, 0, - "tring to unlock the free event mutex %p", m); - return NGX_ERROR; - } - - ngx_log_debug2(NGX_LOG_DEBUG_EVENT, log, 0, - "unlock event mutex %p, next event: %p", m, m->events); - - m->lock = 0; - - if (m->events) { - ev = m->events; - m->events = ev->next; - - ev->next = ngx_posted_events; - ngx_posted_events = ev; - } - - return NGX_OK; -}

@@ -1,307 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) Nginx, Inc. - */ - - -#include <ngx_config.h> -#include <ngx_core.h> -#include <ngx_http.h> - - - -static int ngx_http_busy_lock_look_cacheable(ngx_http_busy_lock_t *bl, - ngx_http_busy_lock_ctx_t *bc, - int lock); - - -int ngx_http_busy_lock(ngx_http_busy_lock_t *bl, ngx_http_busy_lock_ctx_t *bc) -{ - if (bl->busy < bl->max_busy) { - bl->busy++; - - if (bc->time) { - bc->time = 0; - bl->waiting--; - } - - return NGX_OK; - } - - if (bc->time) { - if (bc->time < bl->timeout) { - ngx_add_timer(bc->event, 1000); - return NGX_AGAIN; - } - - bl->waiting--; - return NGX_DONE; - - } - - if (bl->timeout == 0) { - return NGX_DONE; - } - - if (bl->waiting < bl->max_waiting) { - bl->waiting++; - -#if 0 - ngx_add_timer(bc->event, 1000); - bc->event->event_handler = bc->event_handler; -#endif - - /* TODO: ngx_handle_level_read_event() */ - - return NGX_AGAIN; - } - - return NGX_ERROR; -} - - -int ngx_http_busy_lock_cacheable(ngx_http_busy_lock_t *bl, - ngx_http_busy_lock_ctx_t *bc, int lock) -{ - int rc; - - rc = ngx_http_busy_lock_look_cacheable(bl, bc, lock); - - ngx_log_debug3(NGX_LOG_DEBUG_HTTP, bc->event->log, 0, - "http busylock: %d w:%d mw::%d", - rc, bl->waiting, bl->max_waiting); - - if (rc == NGX_OK) { /* no the same request, there's free slot */ - return NGX_OK; - } - - if (rc == NGX_ERROR && !lock) { /* no the same request, no free slot */ - return NGX_OK; - } - - /* rc == NGX_AGAIN: the same request */ - - if (bc->time) { - if (bc->time < bl->timeout) { - ngx_add_timer(bc->event, 1000); - return NGX_AGAIN; - } - - bl->waiting--; - return NGX_DONE; - - } - - if (bl->timeout == 0) { - return NGX_DONE; - } - - if (bl->waiting < bl->max_waiting) { -#if 0 - bl->waiting++; - ngx_add_timer(bc->event, 1000); - bc->event->event_handler = bc->event_handler; -#endif - - /* TODO: ngx_handle_level_read_event() */ - - return NGX_AGAIN; - } - - return NGX_ERROR; -} - - -void ngx_http_busy_unlock(ngx_http_busy_lock_t *bl, - ngx_http_busy_lock_ctx_t *bc) -{ - if (bl == NULL) { - return; - } - - if (bl->md5) { - bl->md5_mask[bc->slot / 8] &= ~(1 << (bc->slot & 7)); - bl->cacheable--; - } - - bl->busy--; -} - - -static int ngx_http_busy_lock_look_cacheable(ngx_http_busy_lock_t *bl, - ngx_http_busy_lock_ctx_t *bc, - int lock) -{ - int i, b, cacheable, free; - u_int mask; - - b = 0; - cacheable = 0; - free = -1; - -#if (NGX_SUPPRESS_WARN) - mask = 0; -#endif - - for (i = 0; i < bl->max_busy; i++) { - - if ((b & 7) == 0) { - mask = bl->md5_mask[i / 8]; - } - - if (mask & 1) { - if (ngx_memcmp(&bl->md5[i * 16], bc->md5, 16) == 0) { - return NGX_AGAIN; - } - cacheable++; - - } else if (free == -1) { - free = i; - } - -#if 1 - if (cacheable == bl->cacheable) { - if (free == -1 && cacheable < bl->max_busy) { - free = i + 1; - } - - break; - } -#endif - - mask >>= 1; - b++; - } - - if (free == -1) { - return NGX_ERROR; - } - - if (lock) { - if (bl->busy == bl->max_busy) { - return NGX_ERROR; - } - - ngx_memcpy(&bl->md5[free * 16], bc->md5, 16); - bl->md5_mask[free / 8] |= 1 << (free & 7); - bc->slot = free; - - bl->cacheable++; - bl->busy++; - } - - return NGX_OK; -} - - -char *ngx_http_set_busy_lock_slot(ngx_conf_t *cf, ngx_command_t *cmd, - void *conf) -{ - char *p = conf; - - ngx_uint_t i, dup, invalid; - ngx_str_t *value, line; - ngx_http_busy_lock_t *bl, **blp; - - blp = (ngx_http_busy_lock_t **) (p + cmd->offset); - if (*blp) { - return "is duplicate"; - } - - /* ngx_calloc_shared() */ - bl = ngx_pcalloc(cf->pool, sizeof(ngx_http_busy_lock_t)); - if (bl == NULL) { - return NGX_CONF_ERROR; - } - *blp = bl; - - /* ngx_calloc_shared() */ - bl->mutex = ngx_pcalloc(cf->pool, sizeof(ngx_event_mutex_t)); - if (bl->mutex == NULL) { - return NGX_CONF_ERROR; - } - - dup = 0; - invalid = 0; - value = cf->args->elts; - - for (i = 1; i < cf->args->nelts; i++) { - - if (value[i].data[1] != '=') { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "invalid value \"%s\"", value[i].data); - return NGX_CONF_ERROR; - } - - switch (value[i].data[0]) { - - case 'b': - if (bl->max_busy) { - dup = 1; - break; - } - - bl->max_busy = ngx_atoi(value[i].data + 2, value[i].len - 2); - if (bl->max_busy == NGX_ERROR) { - invalid = 1; - break; - } - - continue; - - case 'w': - if (bl->max_waiting) { - dup = 1; - break; - } - - bl->max_waiting = ngx_atoi(value[i].data + 2, value[i].len - 2); - if (bl->max_waiting == NGX_ERROR) { - invalid = 1; - break; - } - - continue; - - case 't': - if (bl->timeout) { - dup = 1; - break; - } - - line.len = value[i].len - 2; - line.data = value[i].data + 2; - - bl->timeout = ngx_parse_time(&line, 1); - if (bl->timeout == (time_t) NGX_ERROR) { - invalid = 1; - break; - } - - continue; - - default: - invalid = 1; - } - - if (dup) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "duplicate value \"%s\"", value[i].data); - return NGX_CONF_ERROR; - } - - if (invalid) { - ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "invalid value \"%s\"", value[i].data); - return NGX_CONF_ERROR; - } - } - - if (bl->timeout == 0 && bl->max_waiting) { - ngx_conf_log_error(NGX_LOG_WARN, cf, 0, - "busy lock waiting is useless with zero timeout, ignoring"); - } - - return NGX_CONF_OK; -}

@@ -1,54 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) Nginx, Inc. - */ - - -#ifndef _NGX_HTTP_BUSY_LOCK_H_INCLUDED_ -#define _NGX_HTTP_BUSY_LOCK_H_INCLUDED_ - - -#include <ngx_config.h> -#include <ngx_core.h> -#include <ngx_event.h> -#include <ngx_http.h> - - -typedef struct { - u_char *md5_mask; - char *md5; - int cacheable; - - int busy; - int max_busy; - - int waiting; - int max_waiting; - - time_t timeout; - - ngx_event_mutex_t *mutex; -} ngx_http_busy_lock_t; - - -typedef struct { - time_t time; - ngx_event_t *event; - void (*event_handler)(ngx_event_t *ev); - u_char *md5; - int slot; -} ngx_http_busy_lock_ctx_t; - - -int ngx_http_busy_lock(ngx_http_busy_lock_t *bl, ngx_http_busy_lock_ctx_t *bc); -int ngx_http_busy_lock_cacheable(ngx_http_busy_lock_t *bl, - ngx_http_busy_lock_ctx_t *bc, int lock); -void ngx_http_busy_unlock(ngx_http_busy_lock_t *bl, - ngx_http_busy_lock_ctx_t *bc); - -char *ngx_http_set_busy_lock_slot(ngx_conf_t *cf, ngx_command_t *cmd, - void *conf); - - -#endif /* _NGX_HTTP_BUSY_LOCK_H_INCLUDED_ */

@@ -1,756 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) Nginx, Inc. - */ - - -#include <ngx_config.h> -#include <ngx_core.h> - -/* - * The threads implementation uses the rfork(RFPROC|RFTHREAD|RFMEM) syscall - * to create threads. All threads use the stacks of the same size mmap()ed - * below the main stack. Thus the current thread id is determined via - * the stack pointer value. - * - * The mutex implementation uses the ngx_atomic_cmp_set() operation - * to acquire a mutex and the SysV semaphore to wait on a mutex and to wake up - * the waiting threads. The light mutex does not use semaphore, so after - * spinning in the lock the thread calls sched_yield(). However the light - * mutexes are intended to be used with the "trylock" operation only. - * The SysV semop() is a cheap syscall, particularly if it has little sembuf's - * and does not use SEM_UNDO. - * - * The condition variable implementation uses the signal #64. - * The signal handler is SIG_IGN so the kill() is a cheap syscall. - * The thread waits a signal in kevent(). The use of the EVFILT_SIGNAL - * is safe since FreeBSD 4.10-STABLE. - * - * This threads implementation currently works on i386 (486+) and amd64 - * platforms only. - */ - - -char *ngx_freebsd_kern_usrstack; -size_t ngx_thread_stack_size; - - -static size_t rz_size; -static size_t usable_stack_size; -static char *last_stack; - -static ngx_uint_t nthreads; -static ngx_uint_t max_threads; - -static ngx_uint_t nkeys; -static ngx_tid_t *tids; /* the threads tids array */ -void **ngx_tls; /* the threads tls's array */ - -/* the thread-safe libc errno */ - -static int errno0; /* the main thread's errno */ -static int *errnos; /* the threads errno's array */ - -int * -__error() -{ - int tid; - - tid = ngx_gettid(); - - return tid ? &errnos[tid - 1] : &errno0; -} - - -/* - * __isthreaded enables the spinlocks in some libc functions, i.e. in malloc() - * and some other places. Nevertheless we protect our malloc()/free() calls - * by own mutex that is more efficient than the spinlock. - * - * _spinlock() is a weak referenced stub in src/lib/libc/gen/_spinlock_stub.c - * that does nothing. - */ - -extern int __isthreaded; - -void -_spinlock(ngx_atomic_t *lock) -{ - ngx_int_t tries; - - tries = 0; - - for ( ;; ) { - - if (*lock) { - if (ngx_ncpu > 1 && tries++ < 1000) { - continue; - } - - sched_yield(); - tries = 0; - - } else { - if (ngx_atomic_cmp_set(lock, 0, 1)) { - return; - } - } - } -} - - -/* - * Before FreeBSD 5.1 _spinunlock() is a simple #define in - * src/lib/libc/include/spinlock.h that zeroes lock. - * - * Since FreeBSD 5.1 _spinunlock() is a weak referenced stub in - * src/lib/libc/gen/_spinlock_stub.c that does nothing. - */ - -#ifndef _spinunlock - -void -_spinunlock(ngx_atomic_t *lock) -{ - *lock = 0; -} - -#endif - - -ngx_err_t -ngx_create_thread(ngx_tid_t *tid, ngx_thread_value_t (*func)(void *arg), - void *arg, ngx_log_t *log) -{ - ngx_pid_t id; - ngx_err_t err; - char *stack, *stack_top; - - if (nthreads >= max_threads) { - ngx_log_error(NGX_LOG_CRIT, log, 0, - "no more than %ui threads can be created", max_threads); - return NGX_ERROR; - } - - last_stack -= ngx_thread_stack_size; - - stack = mmap(last_stack, usable_stack_size, PROT_READ|PROT_WRITE, - MAP_STACK, -1, 0); - - if (stack == MAP_FAILED) { - ngx_log_error(NGX_LOG_ALERT, log, ngx_errno, - "mmap(%p:%uz, MAP_STACK) thread stack failed", - last_stack, usable_stack_size); - return NGX_ERROR; - } - - if (stack != last_stack) { - ngx_log_error(NGX_LOG_ALERT, log, 0, - "stack %p address was changed to %p", last_stack, stack); - return NGX_ERROR; - } - - stack_top = stack + usable_stack_size; - - ngx_log_debug2(NGX_LOG_DEBUG_CORE, log, 0, - "thread stack: %p-%p", stack, stack_top); - - ngx_set_errno(0); - - id = rfork_thread(RFPROC|RFTHREAD|RFMEM, stack_top, - (ngx_rfork_thread_func_pt) func, arg); - - err = ngx_errno; - - if (id == -1) { - ngx_log_error(NGX_LOG_ALERT, log, err, "rfork() failed"); - - } else { - *tid = id; - nthreads = (ngx_freebsd_kern_usrstack - stack_top) - / ngx_thread_stack_size; - tids[nthreads] = id; - - ngx_log_debug1(NGX_LOG_DEBUG_CORE, log, 0, "rfork()ed thread: %P", id); - } - - return err; -} - - -ngx_int_t -ngx_init_threads(int n, size_t size, ngx_cycle_t *cycle) -{ - char *red_zone, *zone; - size_t len; - ngx_int_t i; - struct sigaction sa; - - max_threads = n + 1; - - for (i = 0; i < n; i++) { - ngx_memzero(&sa, sizeof(struct sigaction)); - sa.sa_handler = SIG_IGN; - sigemptyset(&sa.sa_mask); - if (sigaction(NGX_CV_SIGNAL, &sa, NULL) == -1) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, - "sigaction(%d, SIG_IGN) failed", NGX_CV_SIGNAL); - return NGX_ERROR; - } - } - - len = sizeof(ngx_freebsd_kern_usrstack); - if (sysctlbyname("kern.usrstack", &ngx_freebsd_kern_usrstack, &len, - NULL, 0) == -1) - { - ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, - "sysctlbyname(kern.usrstack) failed"); - return NGX_ERROR; - } - - /* the main thread stack red zone */ - rz_size = ngx_pagesize; - red_zone = ngx_freebsd_kern_usrstack - (size + rz_size); - - ngx_log_debug2(NGX_LOG_DEBUG_CORE, cycle->log, 0, - "usrstack: %p red zone: %p", - ngx_freebsd_kern_usrstack, red_zone); - - zone = mmap(red_zone, rz_size, PROT_NONE, MAP_ANON, -1, 0); - if (zone == MAP_FAILED) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno, - "mmap(%p:%uz, PROT_NONE, MAP_ANON) red zone failed", - red_zone, rz_size); - return NGX_ERROR; - } - - if (zone != red_zone) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, 0, - "red zone %p address was changed to %p", red_zone, zone); - return NGX_ERROR; - } - - /* create the thread errno' array */ - - errnos = ngx_calloc(n * sizeof(int), cycle->log); - if (errnos == NULL) { - return NGX_ERROR; - } - - /* create the thread tids array */ - - tids = ngx_calloc((n + 1) * sizeof(ngx_tid_t), cycle->log); - if (tids == NULL) { - return NGX_ERROR; - } - - tids[0] = ngx_pid; - - /* create the thread tls' array */ - - ngx_tls = ngx_calloc(NGX_THREAD_KEYS_MAX * (n + 1) * sizeof(void *), - cycle->log); - if (ngx_tls == NULL) { - return NGX_ERROR; - } - - nthreads = 1; - - last_stack = zone + rz_size; - usable_stack_size = size; - ngx_thread_stack_size = size + rz_size; - - /* allow the spinlock in libc malloc() */ - __isthreaded = 1; - - ngx_threaded = 1; - - return NGX_OK; -} - - -ngx_tid_t -ngx_thread_self(void) -{ - ngx_int_t tid; - - tid = ngx_gettid(); - - if (tids == NULL) { - return ngx_pid; - } - - return tids[tid]; -} - - -ngx_err_t -ngx_thread_key_create(ngx_tls_key_t *key) -{ - if (nkeys >= NGX_THREAD_KEYS_MAX) { - return NGX_ENOMEM; - } - - *key = nkeys++; - - return 0; -} - - -ngx_err_t -ngx_thread_set_tls(ngx_tls_key_t key, void *value) -{ - if (key >= NGX_THREAD_KEYS_MAX) { - return NGX_EINVAL; - } - - ngx_tls[key * NGX_THREAD_KEYS_MAX + ngx_gettid()] = value; - return 0; -} - - -ngx_mutex_t * -ngx_mutex_init(ngx_log_t *log, ngx_uint_t flags) -{ - ngx_mutex_t *m; - union semun op; - - m = ngx_alloc(sizeof(ngx_mutex_t), log); - if (m == NULL) { - return NULL; - } - - m->lock = 0; - m->log = log; - - if (flags & NGX_MUTEX_LIGHT) { - m->semid = -1; - return m; - } - - m->semid = semget(IPC_PRIVATE, 1, SEM_R|SEM_A); - if (m->semid == -1) { - ngx_log_error(NGX_LOG_ALERT, log, ngx_errno, "semget() failed"); - return NULL; - } - - op.val = 0; - - if (semctl(m->semid, 0, SETVAL, op) == -1) { - ngx_log_error(NGX_LOG_ALERT, log, ngx_errno, "semctl(SETVAL) failed"); - - if (semctl(m->semid, 0, IPC_RMID) == -1) { - ngx_log_error(NGX_LOG_ALERT, log, ngx_errno, - "semctl(IPC_RMID) failed"); - } - - return NULL; - } - - return m; -} - - -void -ngx_mutex_destroy(ngx_mutex_t *m) -{ - if (semctl(m->semid, 0, IPC_RMID) == -1) { - ngx_log_error(NGX_LOG_ALERT, m->log, ngx_errno, - "semctl(IPC_RMID) failed"); - } - - ngx_free((void *) m); -} - - -ngx_int_t -ngx_mutex_dolock(ngx_mutex_t *m, ngx_int_t try) -{ - uint32_t lock, old; - ngx_uint_t tries; - struct sembuf op; - - if (!ngx_threaded) { - return NGX_OK; - } - -#if (NGX_DEBUG) - if (try) { - ngx_log_debug2(NGX_LOG_DEBUG_MUTEX, m->log, 0, - "try lock mutex %p lock:%XD", m, m->lock); - } else { - ngx_log_debug2(NGX_LOG_DEBUG_MUTEX, m->log, 0, - "lock mutex %p lock:%XD", m, m->lock); - } -#endif - - old = m->lock; - tries = 0; - - for ( ;; ) { - if (old & NGX_MUTEX_LOCK_BUSY) { - - if (try) { - return NGX_AGAIN; - } - - if (ngx_ncpu > 1 && tries++ < 1000) { - - /* the spinlock is used only on the SMP system */ - - old = m->lock; - continue; - } - - if (m->semid == -1) { - sched_yield(); - - tries = 0; - old = m->lock; - continue; - } - - ngx_log_debug2(NGX_LOG_DEBUG_MUTEX, m->log, 0, - "mutex %p lock:%XD", m, m->lock); - - /* - * The mutex is locked so we increase a number - * of the threads that are waiting on the mutex - */ - - lock = old + 1; - - if ((lock & ~NGX_MUTEX_LOCK_BUSY) > nthreads) { - ngx_log_error(NGX_LOG_ALERT, m->log, ngx_errno, - "%D threads wait for mutex %p, " - "while only %ui threads are available", - lock & ~NGX_MUTEX_LOCK_BUSY, m, nthreads); - ngx_abort(); - } - - if (ngx_atomic_cmp_set(&m->lock, old, lock)) { - - ngx_log_debug2(NGX_LOG_DEBUG_MUTEX, m->log, 0, - "wait mutex %p lock:%XD", m, m->lock); - - /* - * The number of the waiting threads has been increased - * and we would wait on the SysV semaphore. - * A semaphore should wake up us more efficiently than - * a simple sched_yield() or usleep(). - */ - - op.sem_num = 0; - op.sem_op = -1; - op.sem_flg = 0; - - if (semop(m->semid, &op, 1) == -1) { - ngx_log_error(NGX_LOG_ALERT, m->log, ngx_errno, - "semop() failed while waiting on mutex %p", m); - ngx_abort(); - } - - ngx_log_debug2(NGX_LOG_DEBUG_MUTEX, m->log, 0, - "mutex waked up %p lock:%XD", m, m->lock); - - tries = 0; - old = m->lock; - continue; - } - - old = m->lock; - - } else { - lock = old | NGX_MUTEX_LOCK_BUSY; - - if (ngx_atomic_cmp_set(&m->lock, old, lock)) { - - /* we locked the mutex */ - - break; - } - - old = m->lock; - } - - if (tries++ > 1000) { - - ngx_log_debug1(NGX_LOG_DEBUG_MUTEX, m->log, 0, - "mutex %p is contested", m); - - /* the mutex is probably contested so we are giving up now */ - - sched_yield(); - - tries = 0; - old = m->lock; - } - } - - ngx_log_debug2(NGX_LOG_DEBUG_MUTEX, m->log, 0, - "mutex %p is locked, lock:%XD", m, m->lock); - - return NGX_OK; -} - - -void -ngx_mutex_unlock(ngx_mutex_t *m) -{ - uint32_t lock, old; - struct sembuf op; - - if (!ngx_threaded) { - return; - } - - old = m->lock; - - if (!(old & NGX_MUTEX_LOCK_BUSY)) { - ngx_log_error(NGX_LOG_ALERT, m->log, 0, - "trying to unlock the free mutex %p", m); - ngx_abort(); - } - - /* free the mutex */ - -#if 0 - ngx_log_debug2(NGX_LOG_DEBUG_MUTEX, m->log, 0, - "unlock mutex %p lock:%XD", m, old); -#endif - - for ( ;; ) { - lock = old & ~NGX_MUTEX_LOCK_BUSY; - - if (ngx_atomic_cmp_set(&m->lock, old, lock)) { - break; - } - - old = m->lock; - } - - if (m->semid == -1) { - ngx_log_debug1(NGX_LOG_DEBUG_MUTEX, m->log, 0, - "mutex %p is unlocked", m); - - return; - } - - /* check whether we need to wake up a waiting thread */ - - old = m->lock; - - for ( ;; ) { - if (old & NGX_MUTEX_LOCK_BUSY) { - - /* the mutex is just locked by another thread */ - - break; - } - - if (old == 0) { - break; - } - - /* there are the waiting threads */ - - lock = old - 1; - - if (ngx_atomic_cmp_set(&m->lock, old, lock)) { - - /* wake up the thread that waits on semaphore */ - - ngx_log_debug1(NGX_LOG_DEBUG_MUTEX, m->log, 0, - "wake up mutex %p", m); - - op.sem_num = 0; - op.sem_op = 1; - op.sem_flg = 0; - - if (semop(m->semid, &op, 1) == -1) { - ngx_log_error(NGX_LOG_ALERT, m->log, ngx_errno, - "semop() failed while waking up on mutex %p", m); - ngx_abort(); - } - - break; - } - - old = m->lock; - } - - ngx_log_debug1(NGX_LOG_DEBUG_MUTEX, m->log, 0, - "mutex %p is unlocked", m); - - return; -} - - -ngx_cond_t * -ngx_cond_init(ngx_log_t *log) -{ - ngx_cond_t *cv; - - cv = ngx_alloc(sizeof(ngx_cond_t), log); - if (cv == NULL) { - return NULL; - } - - cv->signo = NGX_CV_SIGNAL; - cv->tid = -1; - cv->log = log; - cv->kq = -1; - - return cv; -} - - -void -ngx_cond_destroy(ngx_cond_t *cv) -{ - if (close(cv->kq) == -1) { - ngx_log_error(NGX_LOG_ALERT, cv->log, ngx_errno, - "kqueue close() failed"); - } - - ngx_free(cv); -} - - -ngx_int_t -ngx_cond_wait(ngx_cond_t *cv, ngx_mutex_t *m) -{ - int n; - ngx_err_t err; - struct kevent kev; - struct timespec ts; - - if (cv->kq == -1) { - - /* - * We have to add the EVFILT_SIGNAL filter in the rfork()ed thread. - * Otherwise the thread would not get a signal event. - * - * However, we have not to open the kqueue in the thread, - * it is simply handy do it together. - */ - - cv->kq = kqueue(); - if (cv->kq == -1) { - ngx_log_error(NGX_LOG_ALERT, cv->log, ngx_errno, "kqueue() failed"); - return NGX_ERROR; - } - - ngx_log_debug2(NGX_LOG_DEBUG_CORE, cv->log, 0, - "cv kq:%d signo:%d", cv->kq, cv->signo); - - kev.ident = cv->signo; - kev.filter = EVFILT_SIGNAL; - kev.flags = EV_ADD; - kev.fflags = 0; - kev.data = 0; - kev.udata = NULL; - - ts.tv_sec = 0; - ts.tv_nsec = 0; - - if (kevent(cv->kq, &kev, 1, NULL, 0, &ts) == -1) { - ngx_log_error(NGX_LOG_ALERT, cv->log, ngx_errno, "kevent() failed"); - return NGX_ERROR; - } - - cv->tid = ngx_thread_self(); - } - - ngx_mutex_unlock(m); - - ngx_log_debug3(NGX_LOG_DEBUG_CORE, cv->log, 0, - "cv %p wait, kq:%d, signo:%d", cv, cv->kq, cv->signo); - - for ( ;; ) { - n = kevent(cv->kq, NULL, 0, &kev, 1, NULL); - - ngx_log_debug2(NGX_LOG_DEBUG_CORE, cv->log, 0, - "cv %p kevent: %d", cv, n); - - if (n == -1) { - err = ngx_errno; - ngx_log_error((err == NGX_EINTR) ? NGX_LOG_INFO : NGX_LOG_ALERT, - cv->log, ngx_errno, - "kevent() failed while waiting condition variable %p", - cv); - - if (err == NGX_EINTR) { - break; - } - - return NGX_ERROR; - } - - if (n == 0) { - ngx_log_error(NGX_LOG_ALERT, cv->log, 0, - "kevent() returned no events " - "while waiting condition variable %p", - cv); - continue; - } - - if (kev.filter != EVFILT_SIGNAL) { - ngx_log_error(NGX_LOG_ALERT, cv->log, 0, - "kevent() returned unexpected events: %d " - "while waiting condition variable %p", - kev.filter, cv); - continue; - } - - if (kev.ident != (uintptr_t) cv->signo) { - ngx_log_error(NGX_LOG_ALERT, cv->log, 0, - "kevent() returned unexpected signal: %d ", - "while waiting condition variable %p", - kev.ident, cv); - continue; - } - - break; - } - - ngx_log_debug1(NGX_LOG_DEBUG_CORE, cv->log, 0, "cv %p is waked up", cv); - - ngx_mutex_lock(m); - - return NGX_OK; -} - - -ngx_int_t -ngx_cond_signal(ngx_cond_t *cv) -{ - ngx_err_t err; - - ngx_log_debug3(NGX_LOG_DEBUG_CORE, cv->log, 0, - "cv %p to signal %P %d", - cv, cv->tid, cv->signo); - - if (cv->tid == -1) { - return NGX_OK; - } - - if (kill(cv->tid, cv->signo) == -1) { - - err = ngx_errno; - - ngx_log_error(NGX_LOG_ALERT, cv->log, err, - "kill() failed while signaling condition variable %p", cv); - - if (err == NGX_ESRCH) { - cv->tid = -1; - } - - return NGX_ERROR; - } - - ngx_log_debug1(NGX_LOG_DEBUG_CORE, cv->log, 0, "cv %p is signaled", cv); - - return NGX_OK; -}

@@ -1,122 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) Nginx, Inc. - */ - - -#ifndef _NGX_FREEBSD_RFORK_THREAD_H_INCLUDED_ -#define _NGX_FREEBSD_RFORK_THREAD_H_INCLUDED_ - - -#include <sys/ipc.h> -#include <sys/sem.h> -#include <sched.h> - -typedef pid_t ngx_tid_t; - -#define ngx_log_pid ngx_thread_self() -#define ngx_log_tid 0 - -#define NGX_TID_T_FMT "%P" - - -#define NGX_MUTEX_LIGHT 1 - -#define NGX_MUTEX_LOCK_BUSY 0x80000000 - -typedef volatile struct { - ngx_atomic_t lock; - ngx_log_t *log; - int semid; -} ngx_mutex_t; - - -#define NGX_CV_SIGNAL 64 - -typedef struct { - int signo; - int kq; - ngx_tid_t tid; - ngx_log_t *log; -} ngx_cond_t; - - -#define ngx_thread_sigmask(how, set, oset) \ - (sigprocmask(how, set, oset) == -1) ? ngx_errno : 0 - -#define ngx_thread_sigmask_n "sigprocmask()" - -#define ngx_thread_join(t, p) - -#define ngx_setthrtitle(n) setproctitle(n) - - -extern char *ngx_freebsd_kern_usrstack; -extern size_t ngx_thread_stack_size; - - -static ngx_inline ngx_int_t -ngx_gettid(void) -{ - char *sp; - - if (ngx_thread_stack_size == 0) { - return 0; - } - -#if ( __i386__ ) - - __asm__ volatile ("mov %%esp, %0" : "=q" (sp)); - -#elif ( __amd64__ ) - - __asm__ volatile ("mov %%rsp, %0" : "=q" (sp)); - -#else - -#error "rfork()ed threads are not supported on this platform" - -#endif - - return (ngx_freebsd_kern_usrstack - sp) / ngx_thread_stack_size; -} - - -ngx_tid_t ngx_thread_self(void); - - -typedef ngx_uint_t ngx_tls_key_t; - -#define NGX_THREAD_KEYS_MAX 16 - -extern void **ngx_tls; - -ngx_err_t ngx_thread_key_create(ngx_tls_key_t *key); -#define ngx_thread_key_create_n "the tls key creation" - -ngx_err_t ngx_thread_set_tls(ngx_tls_key_t key, void *value); -#define ngx_thread_set_tls_n "the tls key setting" - - -static void * -ngx_thread_get_tls(ngx_tls_key_t key) -{ - if (key >= NGX_THREAD_KEYS_MAX) { - return NULL; - } - - return ngx_tls[key * NGX_THREAD_KEYS_MAX + ngx_gettid()]; -} - - -#define ngx_mutex_trylock(m) ngx_mutex_dolock(m, 1) -#define ngx_mutex_lock(m) (void) ngx_mutex_dolock(m, 0) -ngx_int_t ngx_mutex_dolock(ngx_mutex_t *m, ngx_int_t try); -void ngx_mutex_unlock(ngx_mutex_t *m); - - -typedef int (*ngx_rfork_thread_func_pt)(void *arg); - - -#endif /* _NGX_FREEBSD_RFORK_THREAD_H_INCLUDED_ */

@@ -1,278 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) Nginx, Inc. - */ - - -#include <ngx_config.h> -#include <ngx_core.h> - - -static ngx_uint_t nthreads; -static ngx_uint_t max_threads; - - -static pthread_attr_t thr_attr; - - -ngx_err_t -ngx_create_thread(ngx_tid_t *tid, ngx_thread_value_t (*func)(void *arg), - void *arg, ngx_log_t *log) -{ - int err; - - if (nthreads >= max_threads) { - ngx_log_error(NGX_LOG_CRIT, log, 0, - "no more than %ui threads can be created", max_threads); - return NGX_ERROR; - } - - err = pthread_create(tid, &thr_attr, func, arg); - - if (err != 0) { - ngx_log_error(NGX_LOG_ALERT, log, err, "pthread_create() failed"); - return err; - } - - ngx_log_debug1(NGX_LOG_DEBUG_CORE, log, 0, - "thread is created: " NGX_TID_T_FMT, *tid); - - nthreads++; - - return err; -} - - -ngx_int_t -ngx_init_threads(int n, size_t size, ngx_cycle_t *cycle) -{ - int err; - - max_threads = n; - - err = pthread_attr_init(&thr_attr); - - if (err != 0) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, err, - "pthread_attr_init() failed"); - return NGX_ERROR; - } - - err = pthread_attr_setstacksize(&thr_attr, size); - - if (err != 0) { - ngx_log_error(NGX_LOG_ALERT, cycle->log, err, - "pthread_attr_setstacksize() failed"); - return NGX_ERROR; - } - - ngx_threaded = 1; - - return NGX_OK; -} - - -ngx_mutex_t * -ngx_mutex_init(ngx_log_t *log, ngx_uint_t flags) -{ - int err; - ngx_mutex_t *m; - - m = ngx_alloc(sizeof(ngx_mutex_t), log); - if (m == NULL) { - return NULL; - } - - m->log = log; - - err = pthread_mutex_init(&m->mutex, NULL); - - if (err != 0) { - ngx_log_error(NGX_LOG_ALERT, m->log, err, - "pthread_mutex_init() failed"); - return NULL; - } - - return m; -} - - -void -ngx_mutex_destroy(ngx_mutex_t *m) -{ - int err; - - err = pthread_mutex_destroy(&m->mutex); - - if (err != 0) { - ngx_log_error(NGX_LOG_ALERT, m->log, err, - "pthread_mutex_destroy(%p) failed", m); - } - - ngx_free(m); -} - - -void -ngx_mutex_lock(ngx_mutex_t *m) -{ - int err; - - if (!ngx_threaded) { - return; - } - - ngx_log_debug1(NGX_LOG_DEBUG_MUTEX, m->log, 0, "lock mutex %p", m); - - err = pthread_mutex_lock(&m->mutex); - - if (err != 0) { - ngx_log_error(NGX_LOG_ALERT, m->log, err, - "pthread_mutex_lock(%p) failed", m); - ngx_abort(); - } - - ngx_log_debug1(NGX_LOG_DEBUG_MUTEX, m->log, 0, "mutex %p is locked", m); - - return; -} - - -ngx_int_t -ngx_mutex_trylock(ngx_mutex_t *m) -{ - int err; - - if (!ngx_threaded) { - return NGX_OK; - } - - ngx_log_debug1(NGX_LOG_DEBUG_MUTEX, m->log, 0, "try lock mutex %p", m); - - err = pthread_mutex_trylock(&m->mutex); - - if (err == NGX_EBUSY) { - return NGX_AGAIN; - } - - if (err != 0) { - ngx_log_error(NGX_LOG_ALERT, m->log, err, - "pthread_mutex_trylock(%p) failed", m); - ngx_abort(); - } - - ngx_log_debug1(NGX_LOG_DEBUG_MUTEX, m->log, 0, "mutex %p is locked", m); - - return NGX_OK; -} - - -void -ngx_mutex_unlock(ngx_mutex_t *m) -{ - int err; - - if (!ngx_threaded) { - return; - } - - ngx_log_debug1(NGX_LOG_DEBUG_MUTEX, m->log, 0, "unlock mutex %p", m); - - err = pthread_mutex_unlock(&m->mutex); - - if (err != 0) { - ngx_log_error(NGX_LOG_ALERT, m->log, err, - "pthread_mutex_unlock(%p) failed", m); - ngx_abort(); - } - - ngx_log_debug1(NGX_LOG_DEBUG_MUTEX, m->log, 0, "mutex %p is unlocked", m); - - return; -} - - -ngx_cond_t * -ngx_cond_init(ngx_log_t *log) -{ - int err; - ngx_cond_t *cv; - - cv = ngx_alloc(sizeof(ngx_cond_t), log); - if (cv == NULL) { - return NULL; - } - - cv->log = log; - - err = pthread_cond_init(&cv->cond, NULL); - - if (err != 0) { - ngx_log_error(NGX_LOG_ALERT, cv->log, err, - "pthread_cond_init() failed"); - return NULL; - } - - return cv; -} - - -void -ngx_cond_destroy(ngx_cond_t *cv) -{ - int err; - - err = pthread_cond_destroy(&cv->cond); - - if (err != 0) { - ngx_log_error(NGX_LOG_ALERT, cv->log, err, - "pthread_cond_destroy(%p) failed", cv); - } - - ngx_free(cv); -} - - -ngx_int_t -ngx_cond_wait(ngx_cond_t *cv, ngx_mutex_t *m) -{ - int err; - - ngx_log_debug1(NGX_LOG_DEBUG_CORE, cv->log, 0, "cv %p wait", cv); - - err = pthread_cond_wait(&cv->cond, &m->mutex); - - if (err != 0) { - ngx_log_error(NGX_LOG_ALERT, cv->log, err, - "pthread_cond_wait(%p) failed", cv); - return NGX_ERROR; - } - - ngx_log_debug1(NGX_LOG_DEBUG_CORE, cv->log, 0, "cv %p is waked up", cv); - - ngx_log_debug1(NGX_LOG_DEBUG_MUTEX, m->log, 0, "mutex %p is locked", m); - - return NGX_OK; -} - - -ngx_int_t -ngx_cond_signal(ngx_cond_t *cv) -{ - int err; - - ngx_log_debug1(NGX_LOG_DEBUG_CORE, cv->log, 0, "cv %p to signal", cv); - - err = pthread_cond_signal(&cv->cond); - - if (err != 0) { - ngx_log_error(NGX_LOG_ALERT, cv->log, err, - "pthread_cond_signal(%p) failed", cv); - return NGX_ERROR; - } - - ngx_log_debug1(NGX_LOG_DEBUG_CORE, cv->log, 0, "cv %p is signaled", cv); - - return NGX_OK; -}

@@ -1,73 +0,0 @@ - -/* - * Copyright (C) Igor Sysoev - * Copyright (C) Nginx, Inc. - */ - - -#include <sys/syscall.h> -#include <machine/asm.h> - -/* - * rfork_thread(3) - rfork_thread(flags, stack, func, arg); - */ - -#define KERNCALL int $0x80 - -ENTRY(rfork_thread) - push %ebp - mov %esp, %ebp - push %esi - - mov 12(%ebp), %esi # the thread stack address - - sub $4, %esi - mov 20(%ebp), %eax # the thread argument - mov %eax, (%esi) - - sub $4, %esi - mov 16(%ebp), %eax # the thread start address - mov %eax, (%esi) - - push 8(%ebp) # rfork(2) flags - push $0 - mov $SYS_rfork, %eax - KERNCALL - jc error - - cmp $0, %edx - jne child - -parent: - add $8, %esp - pop %esi - leave - ret - -child: - mov %esi, %esp - pop %eax - call *%eax # call a thread start address ... - add $4, %esp - - push %eax - push $0 - mov $SYS_exit, %eax # ... and exit(2) after a thread would return - KERNCALL - -error: - add $8, %esp - pop %esi - leave - PIC_PROLOGUE - - /* libc's cerror: jmp PIC_PLT(HIDENAME(cerror)) */ - - push %eax - call PIC_PLT(CNAME(__error)) - pop %ecx - PIC_EPILOGUE - mov %ecx, (%eax) - mov $-1, %eax - mov $-1, %edx - ret

Changes of Revision 77