diff --git a/deploy/one-click/cubeproxy/docker-compose.yaml.template b/deploy/one-click/cubeproxy/docker-compose.yaml.template index b56d16ae..ae3fcf36 100644 --- a/deploy/one-click/cubeproxy/docker-compose.yaml.template +++ b/deploy/one-click/cubeproxy/docker-compose.yaml.template @@ -6,9 +6,8 @@ services: context: __CUBE_PROXY_BUILD_CONTEXT__ dockerfile: Dockerfile restart: unless-stopped - ports: - - "__CUBE_PROXY_HOST_PORT__:8080" - - "__CUBE_PROXY_HTTP_HOST_PORT__:8081" + network_mode: host volumes: - __CUBE_PROXY_CERT_DIR__:/usr/local/openresty/nginx/certs:ro - __CUBE_PROXY_GLOBAL_CONF__:/usr/local/openresty/nginx/conf/global/global.conf:ro + - __CUBE_PROXY_NGINX_CONF__:/usr/local/openresty/nginx/conf/nginx.conf:ro diff --git a/deploy/one-click/cubeproxy/nginx.conf.template b/deploy/one-click/cubeproxy/nginx.conf.template new file mode 100644 index 00000000..34775391 --- /dev/null +++ b/deploy/one-click/cubeproxy/nginx.conf.template @@ -0,0 +1,181 @@ +user root; +worker_processes 12; +worker_cpu_affinity auto; +worker_rlimit_nofile 100000; + +error_log /data/log/cube-proxy/error.log notice; +daemon off; + +events { + use epoll; + worker_connections 100000; +} + +http { + include mime.types; + lua_package_path "/usr/local/openresty/nginx/lua/?.lua;;"; + default_type application/octet-stream; + + # do NOT change log format + log_format access '$access_time||$host||-||$http_X_Real_IP||$http_x_forwarded_for||$server_addr||' + '$http_x_forwarded_proto||$request_method||$request_uri||$args||$server_protocol||$uri||' + '$status||$body_bytes_sent||$request_time||$connection_requests||' + '$upstream_addr||$upstream_status||$upstream_response_time||$http_user_agent||' + '$http_x_cube_request_id||$http_x_cube_instance_id||$cube_retcode||$backend_ip||$cache_free_space||' + '-||-||-||$http_x_cube_business'; + + access_log /data/log/cube-proxy/access.log access; + + server_tokens off; + server_names_hash_bucket_size 512; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + + keepalive_timeout 75; + keepalive_requests 10000; + + add_header Vary Accept-Encoding; + + ssi off; + ssi_silent_errors on; + ssi_types text/xml application/javascript application/atom+xml application/rss+xml; + + gzip on; + gzip_vary on; + gzip_http_version 1.0; + gzip_min_length 1000; + gzip_proxied any; + gzip_buffers 4 64k; + gzip_comp_level 3; + gzip_types text/plain text/css text/xml application/x-javascript application/xml application/json; + gzip_disable "MSIE [1-6]\."; + + output_buffers 16 64k; + postpone_output 1460; + + log_subrequest on; + + large_client_header_buffers 4 512k; + client_body_buffer_size 1M; + client_max_body_size 256M; + proxy_buffering on; + proxy_buffer_size 16k; + proxy_buffers 8 64k; + proxy_busy_buffers_size 128k; + proxy_intercept_errors on; + + proxy_temp_file_write_size 128k; + proxy_temp_path /data/; + proxy_cache_path /cache levels=2:2 keys_zone=cache:128m max_size=512m inactive=15m; + proxy_next_upstream_tries 2; + proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504; + + underscores_in_headers on; + + # the backend may be a hostproxy, or it may be a host + upstream backend { + server 0.0.0.1:1234; + balancer_by_lua_file lua/balancer_phase.lua; + keepalive 1500; + keepalive_timeout 80; + } + + lua_shared_dict local_cache 500m; + lua_shared_dict faulty_backend 100m; + init_worker_by_lua_file lua/init_worker_phase.lua; + + server { + listen __CUBE_PROXY_HTTP_PORT__ reuseport; + server_name _; + set $cube_proxy_host_ip ""; + location / { + include /usr/local/openresty/nginx/conf/global/global.conf; + set $cube_retcode "310200"; + set $garyscale_test "none"; + set $backend_ip ""; + set $backend_port ""; + set $access_time ""; + set $cache_free_space ""; + set $host_proxy_port __CUBE_PROXY_HTTP_PORT__; + + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_send_timeout 7206s; + proxy_read_timeout 7206s; + proxy_connect_timeout 3s; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + rewrite_by_lua_file lua/rewrite_phase.lua; + + proxy_pass http://backend; + + header_filter_by_lua_file lua/header_filter_phase.lua; + + log_by_lua_file lua/log_phase.lua; + } + } + + server { + listen __CUBE_PROXY_HTTPS_PORT__ ssl reuseport; + server_name _; + set $cube_proxy_host_ip ""; + ssl_certificate /usr/local/openresty/nginx/certs/cube.app+3.pem; + ssl_certificate_key /usr/local/openresty/nginx/certs/cube.app+3-key.pem; + location / { + include /usr/local/openresty/nginx/conf/global/global.conf; + set $cube_retcode "310200"; + set $garyscale_test "none"; + set $backend_ip ""; + set $backend_port ""; + set $access_time ""; + set $cache_free_space ""; + set $host_proxy_port __CUBE_PROXY_HTTPS_PORT__; + + # NOTE(hengqi): Add support for WebSocket + # + # The WebSocket Protocol is defined in RFC 6455, it says: + # Interoperability considerations + # Use of WebSocket requires use of HTTP version 1.1 or higher. + # + # The doc of `proxy_set_header` says: + # If the value of a header field is an empty string + # then this field will not be passed to a proxied server + # + # So the `proxy_set_header Upgrade $http_upgrade` should have NO side effects. + # + # References: + # - https://datatracker.ietf.org/doc/html/rfc6455 + # - https://datatracker.ietf.org/doc/html/rfc2616#section-14.10 + # - https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Connection + # - https://nginx.org/en/docs/http/websocket.html + # - https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_http_version + # - https://nginx.org/en/docs/http/ngx_http_proxy_module.html#proxy_set_header + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + proxy_send_timeout 7206s; + proxy_read_timeout 7206s; + proxy_connect_timeout 3s; + # proxy_socket_keepalive may cause frequent vm exit + # proxy_socket_keepalive on; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + rewrite_by_lua_file lua/rewrite_phase.lua; + + proxy_pass http://backend; + + header_filter_by_lua_file lua/header_filter_phase.lua; + + log_by_lua_file lua/log_phase.lua; + } + } +} diff --git a/deploy/one-click/env.example b/deploy/one-click/env.example index 49797232..e6ca96cc 100644 --- a/deploy/one-click/env.example +++ b/deploy/one-click/env.example @@ -77,9 +77,10 @@ CUBE_SANDBOX_MYSQL_PASSWORD=cube_pass CUBE_PROXY_ENABLE=1 CUBE_PROXY_IMAGE_TAG=cube-proxy:one-click CUBE_PROXY_CONTAINER_NAME=cube-proxy -CUBE_PROXY_HOST_PORT=443 +CUBE_PROXY_HTTPS_PORT=443 +CUBE_PROXY_HTTP_PORT=80 CUBE_PROXY_CERT_DIR="${ONE_CLICK_INSTALL_PREFIX}/cubeproxy/certs" -CUBE_PROXY_REDIS_IP="${CUBE_SANDBOX_NODE_IP:-}" +CUBE_PROXY_REDIS_IP=127.0.0.1 CUBE_PROXY_DNS_ENABLE=1 CUBE_PROXY_DNS_ANSWER_IP="${CUBE_SANDBOX_NODE_IP:-}" CUBE_PROXY_COREDNS_BIND_ADDR=127.0.0.54 diff --git a/deploy/one-click/scripts/one-click/up-cube-proxy.sh b/deploy/one-click/scripts/one-click/up-cube-proxy.sh index 94fbfd47..d6c41dbc 100644 --- a/deploy/one-click/scripts/one-click/up-cube-proxy.sh +++ b/deploy/one-click/scripts/one-click/up-cube-proxy.sh @@ -26,12 +26,12 @@ COMPOSE_FILE="${PROXY_DIR}/docker-compose.yaml" CUBE_PROXY_IMAGE_TAG="${CUBE_PROXY_IMAGE_TAG:-cube-proxy:one-click}" CUBE_PROXY_CONTAINER_NAME="${CUBE_PROXY_CONTAINER_NAME:-cube-proxy}" -CUBE_PROXY_HOST_PORT="${CUBE_PROXY_HOST_PORT:-443}" -CUBE_PROXY_HTTP_HOST_PORT="${CUBE_PROXY_HTTP_HOST_PORT:-80}" CUBE_SANDBOX_NODE_IP="${CUBE_SANDBOX_NODE_IP:-}" -CUBE_PROXY_REDIS_IP="${CUBE_PROXY_REDIS_IP:-${CUBE_SANDBOX_NODE_IP}}" +CUBE_PROXY_REDIS_IP="${CUBE_PROXY_REDIS_IP:-127.0.0.1}" CUBE_PROXY_REDIS_PORT="${CUBE_PROXY_REDIS_PORT:-${CUBE_SANDBOX_REDIS_PORT:-6379}}" CUBE_PROXY_REDIS_PASSWORD="${CUBE_PROXY_REDIS_PASSWORD:-${CUBE_SANDBOX_REDIS_PASSWORD:-ceuhvu123}}" +CUBE_PROXY_HTTPS_PORT="${CUBE_PROXY_HTTPS_PORT:-443}" +CUBE_PROXY_HTTP_PORT="${CUBE_PROXY_HTTP_PORT:-80}" MKCERT_BUNDLED_BIN="${TOOLBOX_ROOT}/support/bin/mkcert" ensure_dir "${PROXY_DIR}" @@ -84,14 +84,22 @@ sed \ -e "s/__CUBE_PROXY_HOST_IP__/$(escape_sed "${CUBE_SANDBOX_NODE_IP}")/g" \ "${GLOBAL_TEMPLATE}" > "${GLOBAL_CONF}" +NGINX_TEMPLATE="${PROXY_DIR}/nginx.conf.template" +NGINX_CONF="${PROXY_DIR}/nginx.conf" +if [[ -f "${NGINX_TEMPLATE}" ]]; then + sed \ + -e "s/__CUBE_PROXY_HTTPS_PORT__/$(escape_sed "${CUBE_PROXY_HTTPS_PORT}")/g" \ + -e "s/__CUBE_PROXY_HTTP_PORT__/$(escape_sed "${CUBE_PROXY_HTTP_PORT}")/g" \ + "${NGINX_TEMPLATE}" > "${NGINX_CONF}" +fi + sed \ -e "s#__CUBE_PROXY_IMAGE__#$(escape_sed "${CUBE_PROXY_IMAGE_TAG}")#g" \ -e "s#__CUBE_PROXY_CONTAINER_NAME__#$(escape_sed "${CUBE_PROXY_CONTAINER_NAME}")#g" \ -e "s#__CUBE_PROXY_BUILD_CONTEXT__#$(escape_sed "${BUILD_CONTEXT_DIR}")#g" \ - -e "s#__CUBE_PROXY_HOST_PORT__#$(escape_sed "${CUBE_PROXY_HOST_PORT}")#g" \ - -e "s#__CUBE_PROXY_HTTP_HOST_PORT__#$(escape_sed "${CUBE_PROXY_HTTP_HOST_PORT}")#g" \ -e "s#__CUBE_PROXY_CERT_DIR__#$(escape_sed "${CERT_DIR}")#g" \ -e "s#__CUBE_PROXY_GLOBAL_CONF__#$(escape_sed "${GLOBAL_CONF}")#g" \ + -e "s#__CUBE_PROXY_NGINX_CONF__#$(escape_sed "${NGINX_CONF}")#g" \ "${COMPOSE_TEMPLATE}" > "${COMPOSE_FILE}" compose_run down --remove-orphans >/dev/null 2>&1 || true @@ -108,11 +116,11 @@ done [[ "${state:-}" == "running" ]] || die "cube proxy container failed to start" for _ in {1..30}; do - if ss -lnt "( sport = :${CUBE_PROXY_HOST_PORT} )" | rg -q ":${CUBE_PROXY_HOST_PORT}"; then - log "cube proxy listening on ${CUBE_PROXY_HOST_PORT}" + if ss -lnt "( sport = :${CUBE_PROXY_HTTPS_PORT} )" | rg -q ":${CUBE_PROXY_HTTPS_PORT}"; then + log "cube proxy listening on ${CUBE_PROXY_HTTPS_PORT}" exit 0 fi sleep 2 done -die "cube proxy port ${CUBE_PROXY_HOST_PORT} did not become ready" +die "cube proxy port ${CUBE_PROXY_HTTPS_PORT} did not become ready" diff --git a/deploy/one-click/support/docker-compose.yaml.template b/deploy/one-click/support/docker-compose.yaml.template index e106e588..da812c3a 100644 --- a/deploy/one-click/support/docker-compose.yaml.template +++ b/deploy/one-click/support/docker-compose.yaml.template @@ -9,7 +9,7 @@ services: MYSQL_USER: "__MYSQL_USER__" MYSQL_PASSWORD: "__MYSQL_PASSWORD__" ports: - - "__MYSQL_PORT__:3306" + - "127.0.0.1:__MYSQL_PORT__:3306" command: - --default-authentication-plugin=mysql_native_password - --skip-name-resolve @@ -35,7 +35,7 @@ services: - --requirepass - "__REDIS_PASSWORD__" ports: - - "__REDIS_PORT__:6379" + - "127.0.0.1:__REDIS_PORT__:6379" healthcheck: test: [