DaemonForums  

Go Back   DaemonForums > Miscellaneous > Programming

Programming C, bash, Python, Perl, PHP, Java, you name it.

Reply
 
Thread Tools Display Modes
  #1   (View Single Post)  
Old 11th May 2020
CiotBSD CiotBSD is offline
c107:b5d::
 
Join Date: Jun 2019
Location: Under /
Posts: 108
Default [httpd+slowcgi] Script SGI : for static file

Hi,

The context is serve compressed static files with httpd and slowcgi servers, on OpenBSD.

If httpd can serve static files, it can not deliver precompressed static files (deflate, gzip or brotli). slowcgi seconds httpd to delivre thoses compressed static files, by CGI script.

Problem is not to configure httpd to deserve files. (OpenBSD stable 6.6)
(the config is explain into this article: sbw : save bandwidth cgi for httpd

I modify this code, as:
Code:
#!/bin/sh
#set -x
set -e
#set -u
### 
# sbw.cgi - 0.5
# cgi script to serve gzipped file if available with httpd.
# 
# Author: prx <prx@ybad.name>
#   parts stolen from romanzolotarev here : 
#   https://www.romanzolotarev.com/bin/form
#
# Collaborator: PengouinBSD <dev+bsd@huc.fr.eu.org>
#	add brotli support
#	add header Content-Length, Last-Modified, Transfer-Encoding
# 
# Licence: MIT
#
# Description :
# When a file is requested, we try to deliver the gzipped file if it is present.
# i.e. index.html.gz is send instead of plain index.html
# See README
#
# chroot dependencies:
# in /var/www/bin: cat date sh sha256
# in /var/www/usr/bin: logger stat
# in /var/www/usr/lib: libc.so.* - need for stat
# in /var/www/usr/libexec: ld.so - need for stat
#
# CHANGELOG:
# Add ETag header support for better cache.
# TODO :
# * cat replacement ?
###

#PATH_INFO="index.html"
: "${cachecontrol:=3600}"
: "${chroot:=/var/www}"
: "${HTTP_IF_NONE_MATCH:=}"
: "${PATH_INFO?PATH_INFO environement variable must be set with the relative path from realroot}"
: "${realroot:=.}"

check_bin() {
	for bin in cat date sh sha256; do
		test -x "/bin/${bin}" || http500 "${bin}"
	done
	
	for bin in logger stat; do
		test -x "/usr/bin/${bin}" || http500 "${bin}"
	done
}

check_deps() {
	check_bin
	check_lib
}

check_encoding() {
    test -z "${HTTP_ACCEPT_ENCODING}" || encode=1
}

check_error() {
	if [ "${error}" -eq 0 ]; then error=200; fi
}

check_file() {
	if [ -d "${filename}" ]; then
		filename="${realroot}${PATH_INFO}index.html"
	fi
}

check_lib() {
	test -f /usr/lib/libc.so.* || http500 "/usr/lib/libc.so.*"
	test -f /usr/libexec/ld.so || http500 "/usr/libexec/ld.so"
}

check_non_match() {
    test -z "${HTTP_IF_NONE_MATCH}" && return
    if [ "${etag}" = "${HTTP_IF_NONE_MATCH}" ]; then error=304; fi
}

content_type() {
	local file="$1"
	
	case "${file}" in
		*atom.xml|*.atom) type="application/atom+xml" ;;
		*.css)  type="text/css" ;;
		*.html) type="text/html; charset=utf-8" ;;
		*.js)   type="application/javascript" ;;
		*.json)	type="application/json" ;;
		*rss.xml)	type="application/rss+xml" ;;
		*.svg)  type="image/svg+xml" ;;
		*.txt)  type="text/plain; charset=utf-8" ;;		 
		*.xml)  type="text/xml" ;;	 
	esac
	
	if [ "${type}" = "" ]; then
		logger "$0: Can't obtain mime-type, for this file: '${file}'!"
	fi
}

echo_status() {
	echo "Status: ${title}\n"
}

get_http() {
	local title=""
	mng_title
	
	case "${error}" in
        200) http200 ;;
        304) http304 ;;
        404) http404 ;;
        500) http500 ;;
        *)
			logger "$0: This error '${error}' is invalid!"
			exit 1 
        ;;
    esac
}

html() {
	local name="$1"
	local h1="<h1>$title</h1>"
	local title="<title>$title</title>"
	
	local html_header='<!DOCTYPE html>
<head>
	<meta charset="utf-8">
	<style type="text/css">
	body { background-color: black; color: white; font-family: "Comic Sans MS", "Chalkboard SE", "Comic Neue", sans-serif; }
	h1 { color: red; font-weight: bold; }
	hr { border: 0; border-bottom: 1px dashed; }
	</style>
'
	
	local html_h2b='
</head>
<body>
'
	local html_end='
	<hr>
	<address>OpenBSD httpd</address>
</body>
'

	if [ "${name}" != "" ]; then 
		local p="<p>This dependency lacks: ${name}; please, install-it into the chroot: ${chroot}</p>"
	fi
	
	local html="${html_header}\n${title}\n${html_h2b}\n${h1}\n$p\n${html_end}\n"
	
	echo "Content-Type: text/html; charset=utf-8\n"
	echo "${html}"
}

http200() {
	local type=""
	content_type "${filename}"
	
    echo_status
    if [ "${cachecontrol}" -ge 1 ]; then echo "Cache-Control: max-age=${cachecontrol}\n"; fi
    if [ "${encode}" -eq 1 ]; then echo "Content-Encoding: ${serve}\n"; fi
    if [ "${size}" -ge 1 ]; then echo "Content-Length: ${size}\n"; fi
    if [ "${type}" != "" ]; then echo "Content-Type: ${type}\n"; fi
    if [ "${etag}" != "" ]; then echo "ETag: ${etag}\n"; fi
    if [ "${mtime}" != "" ]; then echo "Last-Modified: ${mtime} GMT\n"; fi
    echo "Transfer-Encoding: chunked\n"
    echo "\n"
    
    #echo "$file"
    cat "$file"
    exit 
}

http304() {
    echo_status
    exit
}

http404() {
    echo_status
    html "${filename}"
    exit
}

http500() {
	local error=500
	local name="$1"
	local title=""
	
	mng_title
	
	echo_status
	
	if [ "${name}" != "" ]; then logger "$0: This bin '${name}' lacks; please intall into chroot: '${chroot}'!"; fi
	html "${name}"
	
	exit
}

main() {
    local encode=0
    local error=0
    local etag=""
    local file=""
	local filename="${realroot}${PATH_INFO}"
	local mtime=""
    local serve=""
    local size=0
    
    check_deps
    
    check_encoding
    
    mng_file

    get_http
}

mng_file() {
	
	check_file

	if [ "${encode}" -eq 1 ]; then
        case "${HTTP_ACCEPT_ENCODING}" in
            *br*)
				file="${filename}.br"
				serve="br"
			;;
            *deflate*|*gzip*)
				file="${filename}.gz"
				serve="gzip"
			;;
            *)
				logger "$0: This encoding '${HTTP_ACCEPT_ENCODING}' seems invalid or not supported!"
				file="${filename}"
				serve=""
            ;; 
        esac
    else
		file="${filename}"
    fi
    
	if [ -f "${file}" ]; then 
        set_etag
        
        check_non_match
        
        set_mtime
        
        set_size      
        
        check_error
    else
        error=404
    fi
}

mng_title() {
	case "${error}" in
		200) title='200 OK' ;;
		304) title='304 Not Modified' ;;
		404) title='404 Not Found' ;;
		500) title='500 Internal Server Error' ;;
	esac
}

set_etag() {
	etag="$(sha256 -q "${file}")"
}

set_mtime() {
	local timestamp
	timestamp="$(stat -f"%m" "${file}")"
	mtime="$(date -r $timestamp +"%x %X")"
}

set_size() {
	size="$(stat -f"%z" "${file}")"
}

main
On my server, the compress static file is delivred as "application/octet-stream"!
It should be delivered according to the type of the uncompressed file.
The script echoes needed headers for 'Content-Encoding', 'Content-Length', 'Content-Type', 'Etag', 'Last-Modified', etc.

Someone can help-me to understand why the instruction `cat $files` on the function http200() send files as data!
(and not display into web client)

The original code seems run correctly!

----

PS: I hope my attempts at explanation are clear.
__________________
GPG:Fingerprint ed25519 : 072A 4DA2 8AFD 868D 74CF 9EA2 B85E 9ADA C377 5E8E
GPG:Fingerprint rsa4096 : 4E0D 4AF7 77F5 0FAE A35D 5B62 D0FF 7361 59BF 1733
Reply With Quote
  #2   (View Single Post)  
Old 15th May 2020
CiotBSD CiotBSD is offline
c107:b5d::
 
Join Date: Jun 2019
Location: Under /
Posts: 108
Default

Hi, I attempt to run httpd and slowcgi without rc tool, but to use option '-d', to obtain more infos (useful?):

About httpd -d:
Code:
# httpd -dvvv
startup
socket_rlimit: max open files 1024
socket_rlimit: max open files 1024
server_privinit: adding server default
socket_rlimit: max open files 1024
server_launch: configuring server default
server_launch: configuring server default
server_launch: running server default
server_launch: running server default
server_launch: configuring server default
server_launch: running server default
server default, client 1 (1 active), 127.0.0.1:37017 -> 127.0.0.1, closed
server default, client 1 (1 active), 127.0.0.1:47661 -> 127.0.0.1, closed
server default, client 2 (1 active), 127.0.0.1:23671 -> 127.0.0.1, closed
server default, client 1 (1 active), 127.0.0.1:9508 -> 127.0.0.1, closed
server default, client 3 (1 active), 127.0.0.1:31610 -> 127.0.0.1, closed
server default, client 4 (1 active), 127.0.0.1:1275 -> 127.0.0.1, closed
server default, client 2 (1 active), 127.0.0.1:38210 -> 127.0.0.1, closed
server default, client 5 (1 active), 127.0.0.1:34807 -> 127.0.0.1, closed
server default, client 6 (1 active), 127.0.0.1:7812 -> 127.0.0.1, closed
server default, client 7 (1 active), 127.0.0.1:20353 -> 127.0.0.1, closed
default 127.0.0.1 - - [15/May/2020:07:37:52 +0200] "GET /ecrit/ HTTP/1.1" 200 0
default 127.0.0.1 - - [15/May/2020:07:37:52 +0200] "GET /ecrit/ HTTP/1.1" 200 0
default 127.0.0.1 - - [15/May/2020:07:38:52 +0200] "<UNKNOWN> " 408 0
server default, client 8 (1 active), 127.0.0.1:9952 -> 127.0.0.1, timeout (408 Request Timeout)
About slowcgi -d:
Code:
# slowcgi -d
slowcgi: sock_user: www
slowcgi: socket: /var/www/run/slowcgi.sock
slowcgi: slowcgi_user: www
slowcgi: chroot: /var/www
slowcgi: inflight incremented, now 1
slowcgi: version: 1
slowcgi: type: 1
slowcgi: requestId: 1
slowcgi: contentLength: 8
slowcgi: paddingLength: 0
slowcgi: reserved: 0
slowcgi: role 1
slowcgi: flags 0
slowcgi: version: 1
slowcgi: type: 4
slowcgi: requestId: 1
slowcgi: contentLength: 821
slowcgi: paddingLength: 0
slowcgi: reserved: 0
slowcgi: env[0], PATH_INFO=/ecrit/
slowcgi: env[1], SCRIPT_NAME=
slowcgi: env[2], SCRIPT_FILENAME=/cgi-bin/stat.cgi
slowcgi: env[3], QUERY_STRING=
slowcgi: env[4], DOCUMENT_ROOT=/cgi-bin/stat.cgi
slowcgi: env[5], DOCUMENT_URI=/ecrit/
slowcgi: env[6], GATEWAY_INTERFACE=CGI/1.1
slowcgi: env[7], HTTP_ACCEPT=*/*
slowcgi: env[8], HTTP_ACCEPT_ENCODING=gzip, deflate
slowcgi: env[9], HTTP_ACCEPT_LANGUAGE=en-US,en;q=0.5
slowcgi: env[10], HTTP_CONNECTION=keep-alive
slowcgi: env[11], HTTP_DNT=1
slowcgi: env[12], HTTP_HOST=localhost:8080
slowcgi: env[13], HTTP_REFERER=http://localhost:8080/ecrit/
slowcgi: env[14], HTTP_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; rv:68.0) Gecko/20100101 Firefox/68.0
slowcgi: env[15], HTTP_X_MOZ=prefetch
slowcgi: env[16], realroot=/htdocs/default/dev
slowcgi: env[17], cachecontrol=86400
slowcgi: env[18], file404=/404.html
slowcgi: env[19], realroot=/htdocs/default/dev
slowcgi: env[20], cachecontrol=86400
slowcgi: env[21], file404=/404.html
slowcgi: env[22], REMOTE_ADDR=127.0.0.1
slowcgi: env[23], REMOTE_PORT=9952
slowcgi: env[24], REQUEST_METHOD=GET
slowcgi: env[25], REQUEST_URI=/ecrit/
slowcgi: env[26], SERVER_ADDR=127.0.0.1
slowcgi: env[27], SERVER_PORT=80
slowcgi: env[28], SERVER_NAME=default
slowcgi: env[29], SERVER_PROTOCOL=HTTP/1.1
slowcgi: env[30], SERVER_SOFTWARE=OpenBSD httpd
slowcgi: version: 1
slowcgi: type: 4
slowcgi: requestId: 1
slowcgi: contentLength: 0
slowcgi: paddingLength: 0
slowcgi: reserved: 0
slowcgi: fork: /cgi-bin/stat.cgi
slowcgi: version: 1
slowcgi: type: 5
slowcgi: requestId: 1
slowcgi: contentLength: 0
slowcgi: paddingLength: 0
slowcgi: reserved: 0
slowcgi: resp version: 1
slowcgi: resp type: 6
slowcgi: resp requestId: 1
slowcgi: resp contentLength: 46
slowcgi: resp paddingLength: 2
slowcgi: resp reserved: 0
slowcgi: resp version: 1
slowcgi: resp type: 6
slowcgi: resp requestId: 1
slowcgi: resp contentLength: 226
slowcgi: resp paddingLength: 6
slowcgi: resp reserved: 0
slowcgi: resp version: 1
slowcgi: resp type: 6
slowcgi: resp requestId: 1
slowcgi: resp contentLength: 4096
slowcgi: resp paddingLength: 0
slowcgi: resp reserved: 0
slowcgi: resp version: 1
slowcgi: resp type: 6
slowcgi: resp requestId: 1
slowcgi: resp contentLength: 4096
slowcgi: resp paddingLength: 0
slowcgi: resp reserved: 0
slowcgi: resp version: 1
slowcgi: resp type: 6
slowcgi: resp requestId: 1
slowcgi: resp contentLength: 4071
slowcgi: resp paddingLength: 1
slowcgi: resp reserved: 0
slowcgi: resp version: 1
slowcgi: resp type: 6
slowcgi: resp requestId: 1
slowcgi: resp contentLength: 0
slowcgi: resp paddingLength: 0
slowcgi: resp reserved: 0
slowcgi: resp version: 1
slowcgi: resp type: 7
slowcgi: resp requestId: 1
slowcgi: resp contentLength: 0
slowcgi: resp paddingLength: 0
slowcgi: resp reserved: 0
slowcgi: wait: /cgi-bin/stat.cgi
slowcgi: resp version: 1
slowcgi: resp type: 3
slowcgi: resp requestId: 1
slowcgi: resp contentLength: 8
slowcgi: resp paddingLength: 0
slowcgi: resp reserved: 0
slowcgi: resp appStatus: 0
slowcgi: resp protocolStatus: 0
slowcgi: inflight incremented, now 1
slowcgi: version: 1
slowcgi: type: 1
slowcgi: requestId: 1
slowcgi: contentLength: 8
slowcgi: paddingLength: 0
slowcgi: reserved: 0
slowcgi: role 1
slowcgi: flags 0
slowcgi: version: 1
slowcgi: type: 4
slowcgi: requestId: 1
slowcgi: contentLength: 905
slowcgi: paddingLength: 0
slowcgi: reserved: 0
slowcgi: env[0], PATH_INFO=/ecrit/
slowcgi: env[1], SCRIPT_NAME=
slowcgi: env[2], SCRIPT_FILENAME=/cgi-bin/stat.cgi
slowcgi: env[3], QUERY_STRING=
slowcgi: env[4], DOCUMENT_ROOT=/cgi-bin/stat.cgi
slowcgi: env[5], DOCUMENT_URI=/ecrit/
slowcgi: env[6], GATEWAY_INTERFACE=CGI/1.1
slowcgi: env[7], HTTP_ACCEPT=text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8
slowcgi: env[8], HTTP_ACCEPT_ENCODING=gzip, deflate
slowcgi: env[9], HTTP_ACCEPT_LANGUAGE=en-US,en;q=0.5
slowcgi: env[10], HTTP_CONNECTION=keep-alive
slowcgi: env[11], HTTP_DNT=1
slowcgi: env[12], HTTP_HOST=localhost:8080
slowcgi: env[13], HTTP_REFERER=http://localhost:8080/ecrit/
slowcgi: env[14], HTTP_UPGRADE_INSECURE_REQUESTS=1
slowcgi: env[15], HTTP_USER_AGENT=Mozilla/5.0 (Windows NT 10.0; rv:68.0) Gecko/20100101 Firefox/68.0
slowcgi: env[16], realroot=/htdocs/default/dev
slowcgi: env[17], cachecontrol=86400
slowcgi: env[18], file404=/404.html
slowcgi: env[19], realroot=/htdocs/default/dev
slowcgi: env[20], cachecontrol=86400
slowcgi: env[21], file404=/404.html
slowcgi: env[22], REMOTE_ADDR=127.0.0.1
slowcgi: env[23], REMOTE_PORT=9952
slowcgi: env[24], REQUEST_METHOD=GET
slowcgi: env[25], REQUEST_URI=/ecrit/
slowcgi: env[26], SERVER_ADDR=127.0.0.1
slowcgi: env[27], SERVER_PORT=80
slowcgi: env[28], SERVER_NAME=default
slowcgi: env[29], SERVER_PROTOCOL=HTTP/1.1
slowcgi: env[30], SERVER_SOFTWARE=OpenBSD httpd
slowcgi: version: 1
slowcgi: type: 4
slowcgi: requestId: 1
slowcgi: contentLength: 0
slowcgi: paddingLength: 0
slowcgi: reserved: 0
slowcgi: fork: /cgi-bin/stat.cgi
slowcgi: version: 1
slowcgi: type: 5
slowcgi: requestId: 1
slowcgi: contentLength: 0
slowcgi: paddingLength: 0
slowcgi: reserved: 0
slowcgi: resp version: 1
slowcgi: resp type: 6
slowcgi: resp requestId: 1
slowcgi: resp contentLength: 92
slowcgi: resp paddingLength: 4
slowcgi: resp reserved: 0
slowcgi: resp version: 1
slowcgi: resp type: 6
slowcgi: resp requestId: 1
slowcgi: resp contentLength: 180
slowcgi: resp paddingLength: 4
slowcgi: resp reserved: 0
slowcgi: resp version: 1
slowcgi: resp type: 6
slowcgi: resp requestId: 1
slowcgi: resp contentLength: 4096
slowcgi: resp paddingLength: 0
slowcgi: resp reserved: 0
slowcgi: resp version: 1
slowcgi: resp type: 6
slowcgi: resp requestId: 1
slowcgi: resp contentLength: 4096
slowcgi: resp paddingLength: 0
slowcgi: resp reserved: 0
slowcgi: resp version: 1
slowcgi: resp type: 6
slowcgi: resp requestId: 1
slowcgi: resp contentLength: 4071
slowcgi: resp paddingLength: 1
slowcgi: resp reserved: 0
slowcgi: resp version: 1
slowcgi: resp type: 6
slowcgi: resp requestId: 1
slowcgi: resp contentLength: 0
slowcgi: resp paddingLength: 0
slowcgi: resp reserved: 0
slowcgi: resp version: 1
slowcgi: resp type: 7
slowcgi: resp requestId: 1
slowcgi: resp contentLength: 0
slowcgi: resp paddingLength: 0
slowcgi: resp reserved: 0
slowcgi: wait: /cgi-bin/stat.cgi
slowcgi: resp version: 1
slowcgi: resp type: 3
slowcgi: resp requestId: 1
slowcgi: resp contentLength: 8
slowcgi: resp paddingLength: 0
slowcgi: resp reserved: 0
slowcgi: resp appStatus: 0
slowcgi: resp protocolStatus: 0
In the facts, slowcgi is called properly, but…
But no indication than help to understand why!?
no informative record/trace of returned headers!
or other relevant and useful information... really frustrating!
__________________
GPG:Fingerprint ed25519 : 072A 4DA2 8AFD 868D 74CF 9EA2 B85E 9ADA C377 5E8E
GPG:Fingerprint rsa4096 : 4E0D 4AF7 77F5 0FAE A35D 5B62 D0FF 7361 59BF 1733
Reply With Quote
  #3   (View Single Post)  
Old 15th May 2020
CiotBSD CiotBSD is offline
c107:b5d::
 
Join Date: Jun 2019
Location: Under /
Posts: 108
Default

oki, with solene@, I modify my script to serve compressed files (deflate, gzip or brotli) on OpenBSD with both servers httpd+slowcgi.

Actual version is here:
https://framagit.org/hucste/tools/-/...lowcgi/sbw.cgi
__________________
GPG:Fingerprint ed25519 : 072A 4DA2 8AFD 868D 74CF 9EA2 B85E 9ADA C377 5E8E
GPG:Fingerprint rsa4096 : 4E0D 4AF7 77F5 0FAE A35D 5B62 D0FF 7361 59BF 1733

Last edited by CiotBSD; 15th May 2020 at 06:34 PM.
Reply With Quote
Reply

Tags
cgi, compress, file, httpd, openbsd

Thread Tools
Display Modes

Posting Rules
You may not post new threads
You may not post replies
You may not post attachments
You may not edit your posts

BB code is On
Smilies are On
[IMG] code is On
HTML code is Off

Forum Jump

Similar Threads
Thread Thread Starter Forum Replies Last Post
httpd: vote to add precompressed static files CiotBSD OpenBSD Security 4 11th May 2020 01:43 PM
How to set static IP under OpenBSD ? bsd007 OpenBSD General 1 5th October 2018 05:19 PM
Static shells? Jautenim OpenBSD General 2 13th September 2015 12:49 AM
FreeCode goes static shep News 0 18th June 2014 09:59 PM
Cannot assign static IP. bsdnewbie999 OpenBSD General 5 21st June 2008 09:50 AM


All times are GMT. The time now is 06:33 PM.


Powered by vBulletin® Version 3.8.4
Copyright ©2000 - 2020, Jelsoft Enterprises Ltd.
Content copyright © 2007-2010, the authors
Daemon image copyright ©1988, Marshall Kirk McKusick