diff --git a/README.md b/README.md index 843db46..d06583c 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ We have some guides and general documentation in the [docs](docs/) directory. Am - [Tests](docs/tests.md) - [HTTP Router](docs/router.md) - [Template engine](docs/template.md) +- [Script integrations](docs/util.md) - [List of security fixes](docs/sec-fixes/) ## Dependencies diff --git a/docs/running.md b/docs/running.md index f6df44b..b6cbc97 100644 --- a/docs/running.md +++ b/docs/running.md @@ -9,3 +9,6 @@ The arg parsing is a bit rudimentary atm. Assume only one option supported per i - `debug` shows stderr (useful for debugging) - `debuggier` shows stderr and calltrace - `shell` drops you into an environment practically equivalent to the runtime +- `utils` lists all available [utilities](./util.md) (scripts which integrate into the + standard HTTP.sh environment, but are strictly for CLI use. Useful for administrative tasks.) +- ` [params]` launches an utility, if it's available diff --git a/docs/template-examples.md b/docs/template-examples.md index 10ab8fb..ce0c13c 100644 --- a/docs/template-examples.md +++ b/docs/template-examples.md @@ -145,6 +145,24 @@ This is very useful for rendering data in tables: ![our example, now rendered as a table](https://f.sakamoto.pl/IwIf39cYw.png) +In this specific example the loop index is readily available; However, if your code iterates over +a list of strings and you'd need to keep track of an additional index variable, the template +engine can do it for you with `{{-index}}`: + +``` + + + + + {{start _list}} + + + + + {{end _list}} +
number
{{-index}}meow
+``` + ### integration with notORM notORM's `data_iter` function works great with nested_add; Body of a callback function can be diff --git a/docs/util.md b/docs/util.md new file mode 100644 index 0000000..b11dc2c --- /dev/null +++ b/docs/util.md @@ -0,0 +1,45 @@ +# utils: integrating scripts with http.sh + +HTTP.sh provides a number of useful APIs designed to make interfacing with HTTP and browsers easier. +Some of those (especially notORM) are also useful in a CLI environment, to help with migrations and +other administrative tasks. + +Utils integrate into HTTP.sh through calling the main script, with the utility name as the first +parameter. So, for `meow.sh` the invocation would be `./http.sh meow`, or `./http.sh meow [params]` +if the util takes any parameters. + +Utils are automagically started within the environment, and otherwise work like normal scripts. +Of note: + +- `$0` is always the `./http.sh` invocation, not your script name +- script name is instead stored in `$HTTPSH_SCRIPTNAME` +- `$@` contain parameters to the utility, not a full invocation. +- at the present, only the `master.sh` config is loaded +- the environment is equivalent to `./http.sh shell` + +A list of utilities can be obtained by calling `./http.sh utils`. + +## Built-in utils + +The following scripts are generic helpers shipped with HTTP.sh. We hope they'll come in handy. + +### notORM + +WIP. Currently can only dump a store out to the terminal: + +```bash +$ ./http.sh notORM dump storage/new.dat +declare -a data=([0]="0" [1]="domi" [2]="http://sdomi.pl/" [3]="" [4]="XOaj44smtzCg1p88fgG7bEnMeTNt361wK8Up4BKEiU747lcNIuMAez60zEiAaALWMwzcQ6" [5]="0" [6]="meow~" [7]="RCszUuBXQI") + (…) +``` + +### bump + +Acknowledges a version after a HTTP.sh update. Takes no parameters. + +## Future plans + +- List a description for each util on the `./http.sh utils` page. This would require storing more + metadata on the utils, TBD. +- more 1st-party utils! +- more work on notORM util, ideally it should allow one to do all actions on the DB diff --git a/http.sh b/http.sh index 2d186c8..34a9cab 100755 --- a/http.sh +++ b/http.sh @@ -15,9 +15,12 @@ setup_config() { echo "cfg[init_version]=$HTTPSH_VERSION" >> "config/master.sh" } -if [[ ! -f "$PWD/http.sh" ]]; then - echo -e "Please run HTTP.sh inside its designated directory\nRunning the script from arbitrary locations isn't supported." - exit 1 +if [[ "${0##*/}" == "http.sh" ]]; then + # make sure that working directory is http.sh root + cd "${0%/*}" +elif [[ ! -f "$PWD/http.sh" ]]; then + echo -e "Could not detect HTTP.sh directory\nPlease run HTTP.sh inside its designated directory" + exit 1 fi source src/version.sh @@ -56,7 +59,12 @@ fi source config/master.sh if [[ "$HTTPSH_VERSION" != "${cfg[init_version]}" ]]; then - echo "WARN: HTTP.sh was updated since this instance was initialized (config v${cfg[init_version]:-(none)}, runtime v$HTTPSH_VERSION). There may be breaking changes. Edit cfg[init_version] in config/master.sh to remove this warning." + echo "WARN: HTTP.sh was updated since this instance was initialized (config v${cfg[init_version]:-(none)}, runtime v$HTTPSH_VERSION). There may be breaking changes. + +Check for breaking changes (announced on IRC and in `git log`), +then use this to ACK the message: + +./http.sh bump" fi while read i; do @@ -99,6 +107,32 @@ if [[ "$1" == 'shell' ]]; then exit 0 fi +if [[ "$1" == "utils" ]]; then # list all available utils + # would be cool to make this more generic + ls "${cfg[namespace]}/util/" "src/util" + exit 0 +fi + +for path in "${cfg[namespace]}/util/" "src/util/"; do + HTTPSH_SCRIPTNAME="$(basename "$1")" + if [[ -x "$path$HTTPSH_SCRIPTNAME.sh" ]]; then + shift + shopt -s extglob + source src/account.sh + source src/mail.sh + source src/mime.sh + source src/misc.sh + source src/notORM.sh + source src/template.sh + source "${cfg[namespace]}/config.sh" + source "$path$HTTPSH_SCRIPTNAME.sh" + exit 0 + elif [[ -f "$path$HTTPSH_SCRIPTNAME.sh" ]]; then + echo "[WARN] util '$1' found, but not executable. Ignoring..." >&2 + fi +done +unset path HTTPSH_SCRIPTNAME + cat <&2 _ _ _______ _______ _____ ______ _ _ | | | |_______|_______| _ \/ ___/| | | | @@ -127,13 +161,19 @@ if [[ -f "${cfg[namespace]}/config.sh" ]]; then unset run_once fi +if [[ "${cfg[ip]}" == *":"* ]]; then + socat_listen="TCP6-LISTEN" +else + socat_listen="TCP-LISTEN" +fi + if [[ ${cfg[socat_only]} == true ]]; then echo "[INFO] listening directly via socat, assuming no ncat available" echo "[HTTP] listening on ${cfg[ip]}:${cfg[port]}" if [[ ${cfg[dbg]} == true ]]; then - socat tcp-listen:${cfg[port]},bind=${cfg[ip]},fork "exec:bash -c \'src/server.sh ${cfg[debuggier]}\'" + socat $socat_listen:${cfg[port]},bind=${cfg[ip]},fork "exec:bash -c \'src/server.sh ${cfg[debuggier]}\'" else - socat tcp-listen:${cfg[port]},bind=${cfg[ip]},fork "exec:bash -c src/server.sh" 2>> /dev/null + socat $socat_listen:${cfg[port]},bind=${cfg[ip]},fork "exec:bash -c src/server.sh" 2>> /dev/null if [[ $? != 0 ]]; then echo "[WARN] socat quit with a non-zero status; Maybe the port is in use?" fi @@ -154,7 +194,7 @@ else ncat -i 600s -l -U "$socket" -c src/server.sh -k 2>> /dev/null done & fi - socat TCP-LISTEN:${cfg[port]},fork,bind=${cfg[ip]} UNIX-CLIENT:$socket & + socat $socat_listen:${cfg[port]},fork,bind=${cfg[ip]} UNIX-CLIENT:$socket & echo "[HTTP] listening on ${cfg[ip]}:${cfg[port]} through '$socket'" fi diff --git a/src/notORM.sh b/src/notORM.sh index 887f741..4ca7ba9 100755 --- a/src/notORM.sh +++ b/src/notORM.sh @@ -190,7 +190,7 @@ data_iter() { [[ ! -f "$1" ]] && return 4 local store="$1" local IFS=$'\n' - local r=2 + local _r=2 if [[ "$2" == '{' ]]; then _data_parse_pairs @@ -216,10 +216,10 @@ data_iter() { done "$callback" # only reached if an entry matched all constraints [[ $? == 255 ]] && return 255 - r=0 + _r=0 done < "$store" - return $r + return $_r } # replace a value in `store` with `array`, filtering by `search`. diff --git a/src/server.sh b/src/server.sh index 6be454e..4810a05 100755 --- a/src/server.sh +++ b/src/server.sh @@ -224,30 +224,33 @@ if [[ "${r[post]}" == true ]] && [[ "${r[status]}" == 200 || "${r[status]}" == # I could have done it as an array, but this solution works, and it's # speedy enough so I don't care. - if [[ $tmpdir ]]; then - declare post_multipart - tmpfile=$(mktemp -p $tmpdir) - dd iflag=fullblock of=$tmpfile ibs=${r[content_length]} count=1 obs=1M + if [[ "$tmpdir" ]]; then + if [[ "${cfg[enable_multipart]}" == true ]]; then + # FIXME: ugly, potentially leaky, potentially dangerous code ahead + declare post_multipart + tmpfile=$(mktemp -p $tmpdir) + dd iflag=fullblock of=$tmpfile ibs="${r[content_length]}" count=1 obs=1M - delimeter_len=$(echo -n "${r[content_boundary]}"$'\015' | wc -c) - boundaries_list=$(echo -ne $(grep $tmpfile -ao -e ${r[content_boundary]} --byte-offset | sed -E 's/:(.*)//g') | sed -E 's/ [0-9]+$//') + delimeter_len=$(echo -n "${r[content_boundary]}"$'\015' | wc -c) + boundaries_list=$(echo -ne $(grep $tmpfile -ao -e "${r[content_boundary]}" --byte-offset | sed -E 's/:(.*)//g') | sed -E 's/ [0-9]+$//') - for i in $boundaries_list; do - tmpout=$(mktemp -p $tmpdir) - dd iflag=fullblock if=$tmpfile ibs=$(($i+$delimeter_len)) obs=1M skip=1 | while true; do - read -r line - if [[ $line == $'\015' ]]; then - cat - > $tmpout - break - fi + for i in $boundaries_list; do + tmpout=$(mktemp -p $tmpdir) + dd iflag=fullblock if=$tmpfile ibs=$(($i+$delimeter_len)) obs=1M skip=1 | while true; do + read -r line + if [[ $line == $'\015' ]]; then + cat - > $tmpout + break + fi + done + length=$(grep $tmpout --byte-offset -ae ${r[content_boundary]} | sed -E 's/:(.*)//' | head -n 1) + outfile=$(mktemp -p $tmpdir) + post_multipart+=($outfile) + dd iflag=fullblock if=$tmpout ibs=$length count=1 obs=1M of=$outfile + rm $tmpout done - length=$(grep $tmpout --byte-offset -ae ${r[content_boundary]} | sed -E 's/:(.*)//' | head -n 1) - outfile=$(mktemp -p $tmpdir) - post_multipart+=($outfile) - dd iflag=fullblock if=$tmpout ibs=$length count=1 obs=1M of=$outfile - rm $tmpout - done - rm $tmpfile + rm $tmpfile + fi else if [[ "${r[content_length]}" ]]; then read -r -N "${r[content_length]}" data diff --git a/src/template.sh b/src/template.sh index e2f072c..dab253f 100755 --- a/src/template.sh +++ b/src/template.sh @@ -8,11 +8,7 @@ function render() { local _tpl_ctrl=$'\02' local tplfile - _template_find_absolute_path "$2" - - if [[ ! "$tplfile" ]]; then - exit 1 # fail hard - fi + _template_find_absolute_path "$2" || exit 1 if [[ "$3" != true ]]; then local template="$(tr -d "${_tpl_newline}${_tpl_ctrl}" < "$tplfile" | sed 's/\&/�UwU�/g')" @@ -27,6 +23,7 @@ function render() { # recursion is currently unsupported here, i feel like it may break things? if [[ "$template" == *'{{#'* && "$3" != true ]]; then local subtemplate= + local _old_tplfile="$tplfile" while read key; do # below check prevents the loop loading itself as a template. # this is possibly not enough to prevent all recursions, but @@ -34,19 +31,20 @@ function render() { local i local IFS='' - _old_tplfile="$tplfile" - _template_find_absolute_path "$key" - if [[ "$(realpath "$tplfile")" == "$_old_tplfile" ]]; then + _template_find_absolute_path "$key" || continue + local tplfile_real="$(realpath "$tplfile")" + + if [[ "$tplfile_real" == "$_old_tplfile" ]]; then subtemplate+="s${_tpl_ctrl}\{\{\#$key\}\}${_tpl_ctrl}I cowardly refuse to endlessly recurse\!${_tpl_ctrl}g;" continue fi # don't even try to include files below httpsh's root - [[ "$(realpath "$tplfile")" != "$(dirname "$(realpath "${cfg[namespace]}")")"* ]] && continue - local input="$(tr -d "${_tpl_ctrl}${_tpl_newline}" < "$tplfile" | sed 's/\&/�UwU�/g')" + [[ "$tplfile_real" != "$(dirname "$(realpath "${cfg[namespace]}")")"* ]] && continue + local input="$(tr -d "${_tpl_ctrl}${_tpl_newline}" < "$tplfile_real" | sed 's/\&/�UwU�/g')" garbage+="$input"$'\n' input="$(tr $'\n' "${_tpl_newline}" <<< "$input")" # for another hack subtemplate+="s${_tpl_ctrl}\{\{\#$key\}\}${_tpl_ctrl}${input}${_tpl_ctrl};" - _template_find_special_uri "$(cat "$tplfile")" + _template_find_special_uri "$(cat "$tplfile_real")" done <<< "$(grep -Poh '{{#\K(.*?)(?=}})' <<< "$template")" buf+="${subtemplate}" @@ -85,7 +83,7 @@ function render() { buf+="s${_tpl_ctrl}\{\{start $key\}\}.*\{\{end $key\}\}${_tpl_ctrl}\{\{$key\}\}${_tpl_ctrl};s${_tpl_ctrl}\{\{$key\}\}${_tpl_ctrl}$(tr -d "${_tpl_ctrl}" <<< "$value" | sed "s${_tpl_ctrl}{{start $key}}${_tpl_ctrl}${_tpl_ctrl};s${_tpl_ctrl}{{end $key}}${_tpl_ctrl}${_tpl_ctrl}")${_tpl_ctrl};" unset "$subtemplate" elif [[ "$key" == "@"* && "${ref["$key"]}" != '' ]]; then - local value="$(tr -d "${_tpl_ctrl}${_tpl_newline}" <<< "${ref["$key"]}" | sed -E 's/\&/�UwU�/g')" + local value="$(tr -d "${_tpl_ctrl}${_tpl_newline}" <<< "${ref["$key"]}" | tr '\n' "${_tpl_newline}" | sed -E 's/\&/�UwU�/g')" buf+="s${_tpl_ctrl}\{\{$key\}\}${_tpl_ctrl}${value}${_tpl_ctrl}g;" elif [[ "$key" == "-index" && "$3" == true ]]; then # foreach index mode buf+="s${_tpl_ctrl}\{\{\-index\}\}${_tpl_ctrl}${_index}${_tpl_ctrl}g;" @@ -140,19 +138,22 @@ function render() { # # _template_find_absolute_path(name) -> $tplfile _template_find_absolute_path() { + unset tplfile if [[ "$1" == /dev/stdin || "$1" == "/dev/fd/"* ]]; then tplfile="$1" + return elif [[ ! "${template_relative_paths}" ]]; then tplfile="${cfg[namespace]}/$1" + return else for (( i=0; i<${#template_relative_paths[@]}; i++ )); do if [[ -f "${template_relative_paths[i]}/$1" ]]; then tplfile="${template_relative_paths[i]}/$1" - break + return fi done fi - + return 1 } _template_uri_list=() diff --git a/src/util/bump.sh b/src/util/bump.sh new file mode 100755 index 0000000..7db026b --- /dev/null +++ b/src/util/bump.sh @@ -0,0 +1,8 @@ +#!/bin/bash +if [[ "$HTTPSH_VERSION" != "${cfg[init_version]}" ]]; then + sed -i '/cfg\[init_version\]=/d' config/master.sh + echo "cfg[init_version]=$HTTPSH_VERSION" >> "config/master.sh" + echo "Version bumped. I hope you checked for breaking changes!" +else + echo "All good! :3" +fi diff --git a/src/util/notORM.sh b/src/util/notORM.sh new file mode 100755 index 0000000..27a9120 --- /dev/null +++ b/src/util/notORM.sh @@ -0,0 +1,11 @@ +#!/bin/bash +if [[ ! "$1" ]]; then + echo "usage: $0 $HTTPSH_SCRIPTNAME [params] + +Action can be one of: + dump - dump the contents of a notORM store" + exit 1 +elif [[ "$1" == dump ]]; then + x() { declare -p data; } + data_iter "$2" { } x +fi diff --git a/src/version.sh b/src/version.sh index c290b98..3285da3 100644 --- a/src/version.sh +++ b/src/version.sh @@ -1,2 +1,2 @@ #!/usr/bin/env bash -HTTPSH_VERSION=0.97.2 +HTTPSH_VERSION=0.97.3