From 59293cc7c60688d5b34a038805a33dbdbd2133d3 Mon Sep 17 00:00:00 2001 From: Michael Reed Date: Sun, 9 Jul 2017 03:31:58 -0400 Subject: www/publish.sh: Make safe to run multiple instances concurrently Instead of always using the same file (temp.md), use a unique temporary file so that multiple instances of publish.sh do not clobber each other's work. --- www/publish.sh | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) (limited to 'www') diff --git a/www/publish.sh b/www/publish.sh index 264c95a1..40140728 100755 --- a/www/publish.sh +++ b/www/publish.sh @@ -2,6 +2,7 @@ # # Copyright (C) 2017 Alyssa Rosenzweig # Copyright (C) 2017 Leah Rowe +# Copyright (C) 2017 Michael Reed # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -21,8 +22,9 @@ set -e printf '%s\n' $1 FILE=${1%.md} +TMPFILE=$(mktemp -t libreboot_www.XXXXXXXXXX) -cat $1 > temp.md +cat "$1" > "$TMPFILE" OPTS="-T Libreboot" @@ -46,29 +48,29 @@ else fi if [[ $FILE = *suppliers ]]; then - printf '\n%s\n' "Edit this page -- Back to previous page" >> temp.md + printf '\n%s\n' "Edit this page -- Back to previous page" >> "$TMPFILE" fi if [ "${FILE}" != "./docs/fdl-1.3" ] && [ "${FILE}" != "./conduct" ]; then - cat footer.md >> temp.md + cat footer.md >> "$TMPFILE" fi # change out .md -> .html -sed -i -e 's/\.md\(#[a-z\-]*\)*)/.html\1)/g' temp.md -sed -i -e 's/\.md\(#[a-z\-]*\)*]/.html\1]/g' temp.md +sed -i -e 's/\.md\(#[a-z\-]*\)*)/.html\1)/g' "$TMPFILE" +sed -i -e 's/\.md\(#[a-z\-]*\)*]/.html\1]/g' "$TMPFILE" # change out .md -> .html -sed -i -e 's/\.md\(#[a-z\-]*\)*)/.html\1)/g' temp.md -sed -i -e 's/\.md\(#[a-z\-]*\)*]/.html\1]/g' temp.md +sed -i -e 's/\.md\(#[a-z\-]*\)*)/.html\1)/g' "$TMPFILE" +sed -i -e 's/\.md\(#[a-z\-]*\)*]/.html\1]/g' "$TMPFILE" # work around issue #2872 -TOC=$(grep -q "^x-toc-enable: true$" temp.md && printf '%s\n' "--toc --toc-depth=2") || TOC="" +TOC=$(grep -q "^x-toc-enable: true$" "$TMPFILE" && printf '%s\n' "--toc --toc-depth=2") || TOC="" # work around heterogenous pandoc versions SMART=$(pandoc -v | grep -q '2\.0' || printf '%s\n' "--smart") || SMART="" # chuck through pandoc -pandoc $TOC $SMART temp.md -s --css /global.css $OPTS \ +pandoc $TOC $SMART "$TMPFILE" -s --css /global.css $OPTS \ --template template.html --metadata return="$RETURN"> $FILE.html # additionally, produce bare file for RSS @@ -76,3 +78,6 @@ pandoc $1 > $FILE.bare.html # generate section title anchors as [link] sed -i -e 's_^\(.*\)_
\3
_' $FILE.html + +# clean up temporary file +rm -f "$TMPFILE" -- cgit v1.2.3-70-g09d2 From ac4ba8372107488ea850a74e42e9c978441c6b67 Mon Sep 17 00:00:00 2001 From: Michael Reed Date: Sun, 9 Jul 2017 03:32:14 -0400 Subject: Build website incrementally (and faster if you have the CPU cores) This is done by replacing www/generate.sh with a Makefile. Benefits: - Makes builds incremental, meaning that only the minimum number of markdown files will be converted to HTML during a build. The previous scheme always generated a new HTML file for every markdown file, which is a big waste of time if only 1 or 2 markdown files have been changed. - Allows for much faster builds through concurrent jobs (e.g., via "make -j4"). On my 4-core machine, my average build time for the website with generate.sh was just over 26 seconds; with "make -j4", it was 13 seconds. - Avoids portability issues with find(1) in generate.sh, which I was encountering on OpenBSD. A note on portability: unlike GNU Make, OpenBSD's Make does not have the "$(shell [commands])" construct, so we don't use that. Instead we use "!= [commands]", which is supported by both. --- www/Makefile | 44 ++++++++++++++++++++++++++++++++++++++++++++ www/generate.sh | 9 --------- www/git.md | 2 +- www/publish.sh | 1 - 4 files changed, 45 insertions(+), 11 deletions(-) create mode 100644 www/Makefile delete mode 100755 www/generate.sh (limited to 'www') diff --git a/www/Makefile b/www/Makefile new file mode 100644 index 00000000..681e3067 --- /dev/null +++ b/www/Makefile @@ -0,0 +1,44 @@ +# Copyright (C) 2017 Michael Reed +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +MD_MAIN != find -L . -name '*.md' -and ! -path '*/news/*' +HTML_MAIN = $(MD_MAIN:.md=.html) + +MD_NEWS != find -L news -name '*.md' -and ! -name index.md +HTML_NEWS = $(MD_NEWS:.md=.html) + + +# news/index.html implies the building of $(HTML_NEWS). +all: $(HTML_MAIN) news/index.html + +.SUFFIXES: .md .html +# Does not apply for news/index.md; see below. +.md.html: + ./publish.sh $< + +# Unlike all the other markdown files, news/index.md does not exist at first: +# it must be generated by index.sh. Also note that index.sh depends on the +# existence of the HTML version of all news items, hence the dependency line +# below. +news/index.md: $(HTML_NEWS) + ./index.sh + +clean: + rm -f $(HTML_MAIN) $(HTML_MAIN:.html=.bare.html) \ + $(HTML_NEWS) $(HTML_NEWS:.html=.bare.html) \ + news/index.md news/index.html news/index.bare.html \ + feed.xml news/feed.xml + +.PHONY: clean diff --git a/www/generate.sh b/www/generate.sh deleted file mode 100755 index 3e54dc13..00000000 --- a/www/generate.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash - -[ "x${DEBUG+set}" = 'xset' ] && set -v -set -e - -find -L . -name '*.html' -and -not -name 'template.html' -delete -find -L . -name '*.md' -exec ./publish.sh {} \; -./index.sh -./publish.sh news/index.md diff --git a/www/git.md b/www/git.md index 914bb5c5..ed3b444b 100644 --- a/www/git.md +++ b/www/git.md @@ -24,7 +24,7 @@ with the following scripts in that directory: - index.sh: generates the news feed (on the News section of the website) - publish.sh: converts an .md file to an .html file -- generate.sh: with calls to index.sh and publish.sh, compiles the entire +- Makefile: with calls to index.sh and publish.sh, compiles the entire Libreboot website Use any standard text editor (e.g. Vim, Emacs, Nano, Gedit) to edit the files, diff --git a/www/publish.sh b/www/publish.sh index 40140728..b1b1b405 100755 --- a/www/publish.sh +++ b/www/publish.sh @@ -20,7 +20,6 @@ [ "x${DEBUG+set}" = 'xset' ] && set -v set -e -printf '%s\n' $1 FILE=${1%.md} TMPFILE=$(mktemp -t libreboot_www.XXXXXXXXXX) -- cgit v1.2.3-70-g09d2 From 73ea470f02ffc256b2abd5956cb38cf6a7ed8330 Mon Sep 17 00:00:00 2001 From: Michael Reed Date: Sun, 9 Jul 2017 04:52:29 -0400 Subject: www/publish.sh: Fix conditionals for file matching Before the Makefile, publish.sh was executed on markdown source files using find(1), which happened like this: ./publish.sh ./index.md Now that we have a Makefile, this happens instead: ./publish.sh index.md Note that the file argument "index.md" in the first and second case both refer to the same file, yet they are different strings. This is important because publish.sh gives index.md (among other files) special treatment, and it does this by string comparison. Unfortunately, only the argument in the first case ("./index.md") will cause publish.sh to give special treatment, while the argumnent in the second case ("index.md") will not. To fix this, make it so that both "./index.md" and "index.md" trigger publish.sh's special handling. This commit also fixes the same issue for "docs/fdl-1.3.md" and "conduct.md". --- www/publish.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'www') diff --git a/www/publish.sh b/www/publish.sh index b1b1b405..9c3a4fcd 100755 --- a/www/publish.sh +++ b/www/publish.sh @@ -27,7 +27,9 @@ cat "$1" > "$TMPFILE" OPTS="-T Libreboot" -if [ "${FILE}" != "./index" ]; then +if [[ $FILE == "index" || $FILE == "./index" ]]; then + OPTS="--css /headercenter.css" +else if [[ $FILE == *suppliers ]] then RETURN="" @@ -42,15 +44,14 @@ if [ "${FILE}" != "./index" ]; then RETURN="Edit this page -- Back to previous index" OPTS="-T Libreboot" fi -else - OPTS="--css /headercenter.css" fi if [[ $FILE = *suppliers ]]; then printf '\n%s\n' "Edit this page -- Back to previous page" >> "$TMPFILE" fi -if [ "${FILE}" != "./docs/fdl-1.3" ] && [ "${FILE}" != "./conduct" ]; then +if [[ $FILE != "./docs/fdl-1.3" && $FILE != "docs/fdl-1.3" && + $FILE != "./conduct" && $FILE != "conduct" ]]; then cat footer.md >> "$TMPFILE" fi -- cgit v1.2.3-70-g09d2