From: Georgi Gerganov <redacted>
Date: Tue, 9 Jan 2024 14:23:05 +0000 (+0200)
Subject: scripts : script to get Paul Graham essays in txt format (#4838)
X-Git-Tag: upstream/0.0.4488~2688
X-Git-Url: https://git.djapps.eu/?a=commitdiff_plain;h=d9653894dffbfd3a58616f31b0967b34faf6f611;p=pkg%2Fggml%2Fsources%2Fllama.cpp

scripts : script to get Paul Graham essays in txt format (#4838)
---

diff --git a/scripts/get-pg.sh b/scripts/get-pg.sh
new file mode 100755
index 00000000..d516db46
--- /dev/null
+++ b/scripts/get-pg.sh
@@ -0,0 +1,47 @@
+#!/bin/bash
+
+function usage {
+    echo "usage: <n>$0"
+    exit 1
+}
+
+function has_cmd {
+    if ! [ -x "$(command -v $1)" ]; then
+        echo "error: $1 is not available" >&2
+        exit 1
+    fi
+}
+
+# check for: curl, html2text, tail, sed, fmt
+has_cmd curl
+has_cmd html2text
+has_cmd tail
+has_cmd sed
+
+if [ $# -ne 1 ]; then
+    usage
+fi
+
+n=$1
+
+# get urls
+urls="$(curl http://www.aaronsw.com/2002/feeds/pgessays.rss | grep html | sed -e "s/.*http/http/" | sed -e "s/html.*/html/" | head -n $n)"
+
+printf "urls:\n%s\n" "$urls"
+
+if [ -f pg.txt ]; then
+    rm pg.txt
+fi
+
+for url in $urls; do
+    echo "processing $url"
+
+    curl -L $url | html2text | tail -n +4 | sed -E "s/^[[:space:]]+//g" | fmt -w 80 >> pg.txt
+
+    # don't flood the server
+    sleep 1
+done
+
+echo "done. data in pg.txt"
+
+exit 0