Ciro Santilli @cirosantilli 35

cia-2010-covert-communication-websites/bigb-to-json cia-2010-covert-communication-websites.bigb

cia-2010-covert-communication-websites/bigb-to-json-merge > tmp.json
mv tmp.json ../media/cia-2010-covert-communication-websites/hits.json

+++ b/cia-2010-covert-communication-websites/cdx-post.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 # Post process the output of cdx.sh to enrich IDs even further, and reconstruct easier to Web Archive inspect domain names.
-grep -P -e '([^,)]+)\)\/\1\.swf|\)/[^/]+.jar|([^,)]+),([^,)]+),([^,)]+)\)/cgi-bin/[^/]+\.cgi' "$1" |
-  sed -r 's/\).*//' | awk -F, '{ printf("%s.%s\n", $2, $1) }' | uniq -c | awk '$1 == 1{ print $2 }' | tee $1.post
+grep -P -e '([^,)]+)\)\/\1\.swf|\)/[^/]+.jar|([^,)]+),([^,)]+),([^,)]+)\)/cgi-bin/[^/]+\.cgi' "$1"|
+  sed -r 's/\).*//' | awk -F, '{ printf("%s.%s\n", $2, $1) }' | uniq -c | awk '{ print $2 }' | tee $1.post

./hupo-cdx-tor.sh out 'news|headline|internationali|mondo|mundo|mondi|iran|today' 2006 2022