#!/usr/local/bin/gawk -bE # # vebug - https://en.wikipedia.org/wiki/User:GreenC_bot/Job_18 # # The MIT License (MIT) # # Copyright (c) August 2019 User:GreenC (en.wikipedia.org) # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN # THE SOFTWARE. BEGIN { BotName = "vebug" } @include "botwiki.awk" @include "library.awk" @include "json.awk" BEGIN { Mode = "bot" # set to "find" and it will search only and exit with a 1 (found something) or 0 (found nothing) # in "find" mode, run via 'project -s' to search local cache for articles containing actionable matches # set to anything else and it will process the article. IGNORECASE = 1 ReSpace = "[\n\r\t]*[ ]*[\n\r\t]*[ ]*[\n\r\t]*" ReSups = "[<]sup[^>]*[>][^<]+[<][/]sup[>]" delete citeTable Optind = Opterr = 1 while ((C = getopt(ARGC, ARGV, "hs:l:n:")) != -1) { opts++ if(C == "s") # -s <file> article.txt source to process. articlename = verifyval(Optarg) if(C == "l") # -l <dir/> Directory where logging is sent.. end with "/" logdir = verifyval(Optarg) if(C == "n") # -n <name> Wikipedia name of article wikiname = verifyval(Optarg) if(C == "h") { usage() exit } } if( ! opts || articlename == "" ) { stdErr("Error in vebug.awk (1)") print "0" exit } if(wikiname == "" || logdir == "") Logfile = "/dev/null" else { if(substr(logdir, length(logdir), 1) != "/") logdir = logdir "/" Logfile = logdir "logvebug" } # Path to data directory ends in "/" DataDir = articlename gsub(regesc3(basename(articlename)) "$", "", DataDir) # Number of changes made to article Count = 0 main() } # # Run the program, save the page to disk # function main( article,articlenew,articlenewname,editsummaryname,bn,plural) { checkexists(articlename, "vebug.awk main()", "exit") article = readfile(articlename) if(length(article) < 10) { print "0" exit } article = deflate(article) articlenew = vebug(article) if(article != articlenew && length(articlenew) > 10 && Count > 0) { articlenew = inflate(articlenew) articlenewname = DataDir "article.vebug.txt" printf("%s", articlenew) > articlenewname close(articlenewname) editsummaryname = DataDir "editsummary.vebug.txt" if(Count > 1) plural = "s" printf("Restore %s cite" plural " deleted by a bug in VisualEditor ([[User:GreenC_bot/Job_18|vebug bot]])", Count) > editsummaryname # Customize the edit summary to be more specific close(editsummaryname) print Count exit } print "0" exit } # # vebug - parse page, load data into citeTable[][], modify page, return it to main() # function vebug(article, c,field,sep,i,field2,sep2,j,d,k,dest,dest2,fieldi,field2txt,setdone,key,vertemp,re,foundit) { # Special case # Regularize <sup>[[User:Claudia Diaz2/sandbox#cite%20note-5|[6]]<nowiki>]</nowiki></sup> c = patsplit(article, field, /[|][[][^]]*[]]{2}[<]nowiki[>][]][<][/]nowiki[>]/, sep) for(k = 1; k <= c; k++) { if(match(reverse(sep[k-1]), /[^[]+[[]{2}/, dest) ) { origcite = field[k] gsub(/<\/?nowiki>/, "", field[k]) field[k] = reverse(dest[0]) field[k] origTable[gsubi("^[<]nowiki[^>]*[>]|[<][/]nowiki[>]", "", field[k])] = reverse(dest[0]) origcite sep[k-1] = gsubs(reverse(dest[0]), "", sep[k-1]) } } article = unpatsplit(field, sep) # Special case # Regularize [[Politics of Venezuela#cite%20note-19|<sup>[1</sup>]] c = patsplit(article, field, /[|][<]sup[>][[][0-9]*[<][/]sup[>][]]{2}/, sep) for(k = 1; k <= c; k++) { if(match(reverse(sep[k-1]), /[^[]+[[]{2}/, dest) ) { origcite = field[k] sub(/[<]sup[>][[][^<]*[<][/]sup[>]/, "[]", field[k]) field[k] = reverse(dest[0]) field[k] origTable[gsubi("^[<]nowiki[^>]*[>]|[<][/]nowiki[>]", "", field[k])] = reverse(dest[0]) origcite sep[k-1] = gsubs(reverse(dest[0]), "", sep[k-1]) } } article = unpatsplit(field, sep) # Special case # Regularize <sup>[[User:Claudia Diaz2/sandbox#cite%20note-5|<nowiki>6]</nowiki>]]</sup> c = patsplit(article, field, /[|][<]nowiki[>][^]]*[]]{1}[<][/]nowiki[>][]]{2}/, sep) for(k = 1; k <= c; k++) { if(match(reverse(sep[k-1]), /[^[]+[[]{2}/, dest) ) { origcite = field[k] sub(/<nowiki>[^]]*[^]]/, "[", field[k]) sub(/<\/nowiki>/, "", field[k]) field[k] = reverse(dest[0]) field[k] origTable[gsubi("^[<]nowiki[^>]*[>]|[<][/]nowiki[>]", "", field[k])] = reverse(dest[0]) origcite sep[k-1] = gsubs(reverse(dest[0]), "", sep[k-1]) } } article = unpatsplit(field, sep) # Special case # Regularize [[2017 Women's March#cite%20note-FT%20100%2C000-13|<span>[13]</span>]] # works for <sup> or <span> split("sup span", tag, " ") for(k = 1; k <= 2; k++) { re = "[<]" tag[k] "[^>]*[>][[][0-9]+[]][<][/]" tag[k] "[>][]]{2}" c = patsplit(article, field, re, sep) for(i = 1; i <= c; i++) { if(match(reverse(sep[i-1]), /[|][^[]+[[]{2}/, dest) ) { origcite = field[i] re = "^[<]" tag[k] "[^>]*[>]|[<][/]" tag[k] "[>]" gsub(re, "", field[i]) field[i] = "<sup>" reverse(dest[0]) field[i] "</sup>" origTable[gsubi("^[<]" tag[k] "[^>]*[>]|[<][/]" tag[k] "[>]","",field[i])] = reverse(dest[0]) origcite sep[i-1] = gsubs(reverse(dest[0]), "", sep[i-1]) } } article = unpatsplit(field,sep) } # Special case # Convert cases not surrounded by <sup> or <span> d = patsplit(article, field2, ReSups, sep2) # Everything already surrounded by sup c = patsplit(reverse(article), field, /[]]{3}[0-9]{0,3}[[][|][0-9]{1,3}[-][^[]+[[]{2}/, sep) for(i = 1; i <= c; i++) { foundit = 0 for(j = 1; j <= d; j++) { if(field2[j] ~ regesc3(reverse(field[i]))) { foundit = 1 } } if( ! foundit) { origTable["<sup>" reverse(field[i]) "</sup>"] = reverse(field[i]) field[i] = ">pus/<" field[i] ">pus<" } } article = reverse(unpatsplit(field, sep)) # Standard: convert cites surrounded by <sup>..</sup> c = patsplit(article, field, ReSups, sep) for(i = 1; i <= c; i++) { if(field[i] !~ /[{][{][^{]*[{][{]/ && field[i] ~ /cite[%]20/) { # skip embeded templates not found by deflate() sendlog(logdir "logsups", wikiname, field[i]) # Encode embedded [0-9] so it can be parsed # <sup>[[Group of Eight#cite%20note-19|[19]]][[Group of Eight#cite%20note-20|[20]]]</sup> --> # <sup>[[Group of Eight#cite%20note-19|VEBUGO19VEBUGC]][[Group of Eight#cite%20note-20|VEBUGO20VEBUGC]]</sup> fieldi = field[i] while(match(fieldi, "[[][0-9]+[]]", dest)) { if(match(dest[0], /[0-9]+/, dest2)) field[i] = gsubs(dest[0], "VEBUGO" dest2[0] "VEBUGC", field[i]) sub("[[][0-9]+[]]", "", fieldi) } # Populate citeTable[][] delete citeTable d = patsplit(field[i], field2, /[[]{2}[^]]+[]]{1,3}/, sep2) for(j = 1; j <= d; j++) { # Decoded field2txt = field2[j] field2txt = gsubs("VEBUGO", "[", field2txt) field2txt = gsubs("VEBUGC", "]", field2txt) citeTable[field2txt]["decoded"] = field2txt key = field2txt # Encoded citeTable[key]["encoded"] = field2[j] if( empty(origTable[key])) origTable[key] = key # Cite number getCiteNumbers(key) if(abort(key, "citenumber")) continue # Primary article getTitle(key, "artprimary") if(abort(key, "artprimary")) continue # Secondary article title eg. Group of Eight getTitle(key, "artsecondary") if(abort(key, "artsecondary")) continue # Time/revision it was last added to primary article getPrimaryRevTime(key) if(abort(key, "artprimaryrevid")) continue if(abort(key, "artprimarytimestamp")) continue # Time/revision it existed in secondary article getSecondaryRevTime(key) if(abort(key, "artsecondaryrevid")) continue if(abort(key, "artsecondarytimestamp")) continue if(abort(key, "artsecondarywikitext")) continue setdone = 0 # if 1, ref is established early in process due to missing data settype = 0 if(! empty(citeTable[key]["citenumbermismatch"])) { split(citeTable[key]["citenumbermismatch"], cites, /[|]/) vertemp = "{{verify source |date=" getDateToday() " |reason=This ref was deleted Special:Diff/" citeTable[key]["artprimaryrevid"] " by a bug in VisualEditor and later identified by a bot. The original cite can be found at Special:Permalink/" citeTable[key]["artsecondaryrevid"] " (or in a rev close to it) in either cite #" cites[1] " or cite #" cites[2] " - find and verify the cite and replace this template with it (1). [[User:GreenC_bot/Job_18]]}}" field2[j] = "<ref>Citation error. See inline comment how to fix. " vertemp "</ref>" settype = 1 setdone = 1 } # Get cite from secondary by its number if( ! setdone) { getSecondaryCiteByNumer(key) if( empty(citeTable[key]["citesecondaryplaintext"]) ) { vertemp = "{{verify source |date=" getDateToday() " |reason=This ref was deleted Special:Diff/" citeTable[key]["artprimaryrevid"] " by a bug in VisualEditor and later identified by a bot. The original cite can be found at Special:Permalink/" citeTable[key]["artsecondaryrevid"] " (or in a rev close to it) as cite #" citeTable[key]["citenumber"] " - find and verify the cite and replace this template with it (2). [[User:GreenC_bot/Job_18]]}}" field2[j] = "<ref>Citation error. See inline comment how to fix. " vertemp "</ref>" settype = 2 setdone = 1 } } # Upload page to wikipedia if( ! setdone) { if( generateCitationsPage(key) == 0) { vertemp = "{{verify source |date=" getDateToday() " |reason=This ref was deleted Special:Diff/" citeTable[key]["artprimaryrevid"] " by a bug in VisualEditor and later restored by a bot in plain-text form. The original cite can be found at Special:Permalink/" citeTable[key]["artsecondaryrevid"] " (or in a rev close to it) as cite #" citeTable[key]["citenumber"] " - find and verify the cite and replace this template with it (3). [[User:GreenC_bot/Job_18]]}}" field2[j] = "<ref>" citeTable[key]["citesecondaryplaintext"] " " vertemp "</ref>" settype = 3 setdone = 1 } } # Parse page if( ! setdone) { parseCitationsPage(key) if( empty(citeTable[key]["citesecondarywikicite"]) ) { vertemp = "{{verify source |date=" getDateToday() " |reason=This ref was deleted Special:Diff/" citeTable[key]["artprimaryrevid"] " by a bug in VisualEditor and later restored by a bot in plain-text form. The original cite can be found at Special:Permalink/" citeTable[key]["artsecondaryrevid"] " (or in a rev close it it) as cite #" citeTable[key]["citenumber"] " - find and verify the cite and replace this template with it (4). [[User:GreenC_bot/Job_18]]}}" field2[j] = "<ref>" citeTable[key]["citesecondaryplaintext"] " " vertemp "</ref>" settype = 4 setdone = 1 } } if( ! setdone) { vertemp = "{{verify source |date=" getDateToday() " |reason=This ref was deleted Special:Diff/" citeTable[key]["artprimaryrevid"] " by a bug in VisualEditor and later restored by a bot from the original cite located at Special:Permalink/" citeTable[key]["artsecondaryrevid"] " cite #" citeTable[key]["citenumber"] " - verify the cite is accurate and delete this template. [[User:GreenC_bot/Job_18]]}}" field2[j] = "<ref>" citeTable[key]["citesecondarywikicite"] " " vertemp "</ref>" sendlog(logdir "logconvert", wikiname, key " ---- " citeTable[key]["citesecondarywikicite"] ) } else { sendlog(logdir "logconvert", wikiname, key " ---- " settype ) } Count++ } field[i] = unpatsplit(field2,sep2) gsub(/^[<]sup[^>]*[>]|[<][/]sup[>]$/, "", field[i]) # Save debuggung info to table.out in data directory saveTable() } } article = unpatsplit(field, sep) # Check for missed sups if(article ~ /[#]cite[%]20/ || article ~ /VEBUG[OC]/ ) { sendlog(logdir "logmissed", wikiname, "Contains cite%20 or VEBUGO . aborting") print article > DataDir "article.abort.txt" Count = 0 # Abort changes to article } return article } # # Get citations numbers # # Populates: # citeTable[key]["citenumber"] (a single cite number best guess, usually the second number) # citeTable[key]["citenumbermismatch"] (two cites numbers sep by "|" if different, otherwise blank) # function getCiteNumbers(key, buf1,buf2,dest) { citeTable[key]["citenumber"] = "" citeTable[key]["citenumbermismatch"] = "" # -55|[note 1]]$ (get "note 1") if(match(reverse(citeTable[key]["decoded"]), /^[]]{2}[^[]*[[][|]/, dest)) { gsub(/^[]]{2,3}|[[][|]$/, "", dest[0]) buf1 = reverse(dest[0]) if(! empty(strip(buf1))) citeTable[key]["citenumber"] = buf1 } # -55|[note 1]]$ (get "55") if(match(reverse(citeTable[key]["decoded"]), /[|][^-]*[^-]/, dest)) { gsub(/^[|]/, "", dest[0]) if(! empty(strip(dest[0]))) buf2 = reverse(dest[0]) } if(! empty(citeTable[key]["citenumber"]) && ! empty(buf2)) { if(buf2 != citeTable[key]["citenumber"]) citeTable[key]["citenumbermismatch"] = buf2 "|" citeTable[key]["citenumber"] } if( ! isanumber(citeTable[key]["citenumber"]) && isanumber(buf2)) citeTable[key]["citenumber"] = buf2 } # # Parse citations page User:GreenC/testcases/iabdebug # # Populate citeTable[key]["citesecondarywikicite"] # function parseCitationsPage(key, begin,i,a,b,fp,np,c,d,e) { citeTable[key]["citesecondarywikicite"] = "" # Convert page to plain-text command = "w3m -dump -cols 10000 'https://en.wikipedia.org/wiki/User:GreenC/testcases/iabdebug'" fp = sys2var(command) # Extract core surrounded by "%%%%" and "^^^^" begin = 0 for(i = 1; i <= splitn(fp, a, i); i++) { if(a[i] ~ /^[%]{4}$/) { begin = 1 continue } if(a[i] ~ /^[\\^]{4}$/) begin = 0 if(begin) np = np "\n" a[i] } # Extract records surrounded by "@@@@" c = split(np, b, /[@]{4}\n/) for(i = 1; i <= c; i++) { if(! empty(strip(b[i]))) { d = split(strip(b[i]), e, /[+]{4}\n/) e[1] = gsubs("^[dead link]", "", e[1]) # print strip(e[1]) " = " citeTable[key]["citesecondaryplaintext"] if(strip(e[1]) == citeTable[key]["citesecondaryplaintext"]) { citeTable[key]["citesecondarywikicite"] = strip(e[2]) break } } } } # # Generate a list of citations in parsable format and upload to User:GreenC/testcases/iabdebug # function generateCitationsPage(key, c,b,i,k,bp,np,status) { bp = "\n<p>\n" np = setdatetype(citeTable[key]["artsecondarywikitext"]) bp # need to set date type so CS1|2 date display is consistent np = np "%%%%" bp c = split(citeTable[key]["artsecondarywikitext"], b, "<ref[^>]*>") for(i = 1; i <= c; i++) { k = strip(substr(b[i], 1, match(b[i], "</ref>") - 1)) if( ! empty(k)) { np = np "@@@@" bp np = np k bp np = np "++++" bp np = np "<nowiki>" k "</nowiki>" bp } } np = np "^^^^" bp # Upload page for(i = 1; i <= 3; i++) { status = sys2varPipe(np, "wikiget -E " shquote("User:GreenC/testcases/iabdebug") " -S " shquote(citeTable[key]["artprimary"] " -/- " citeTable[key]["artsecondary"]) " -P STDIN") if(status ~ "Success" || status ~ "No change") { sleep(5) return 1 } else sleep(5) } sendlog(logdir "syslog", wikiname, "Error: Unable to upload to User page, wikiget returns: " status) return 0 } # # Get plain-text version of given cite number in secondary article # populates citeTable[key]["citesecondaryplaintext"] # on error set to blank # function getSecondaryCiteByNumer(key, command,fp,citenum,a,i) { citeTable[key]["citesecondaryplaintext"] = "" # Plain text of secondary article command = "w3m -dump -cols 10000 " shquote("https://en.wikipedia.org/w/index.php?title=" urlencodeawk(citeTable[key]["artsecondary"]) "&oldid=" citeTable[key]["artsecondaryrevid"] ) fp = sys2var(command) # Get the cite # in plain-text if( int(citeTable[key]["citenumber"]) < 10) citenum = "^[ ]" citeTable[key]["citenumber"] "[.][ ][\\^]" else citenum = "^" citeTable[key]["citenumber"] "[.][ ][\\^]" for(i = 1; i <= splitn(fp, a, i); i++) { if(a[i] ~ citenum) { sub(citenum, "", a[i]) gsub(/[\\^][a-z]{1,3}[ ]/, "", a[i]) a[i] = gsubs("^[dead link]", "", a[i]) citeTable[key]["citesecondaryplaintext"] = strip(a[i]) # break # keep going to get past any duplicates in 'notes' section .. this is imperfect though } } } # # What time and revision was the bug added to the primary article? # populates: # citeTable[key]["artsecondaryrevid"] # citeTable[key]["artsecondarytimestamp"] # citeTable[key]["artsecondarywikitext"] # on error they are blank # function getSecondaryRevTime(key, jsona,i,cont,unixprimary,command,j,arrevid,artimestamp,arcontinue,a,maxrevs,prevcontinue) { citeTable[key]["artsecondaryrevid"] = "" citeTable[key]["artsecondarytimestamp"] = "" citeTable[key]["artsecondarywikitext"] = "" if(! empty(citeTable[key]["artprimarytimestamp"])) unixPrimary = unixTime(citeTable[key]["artprimarytimestamp"]) else return maxrevs = 20 i = 0 cont = 1 while(cont) { i++ if(i > maxrevs) { sendlog(logdir "syslog", wikiname, "Error: Exceeded " maxrevs " API requests in getSecondaryRevTime()") break # sanity break } if(empty(arcontinue["continue"])) command = "wget -q -O- " shquote("https://en.wikipedia.org/w/api.php?action=query&prop=revisions&titles=" urlencodeawk(citeTable[key]["artsecondary"]) "&rvlimit=50&rvslots=main&rvprop=timestamp%7Cuser%7Cids&rvdir=older&format=json") else command = "wget -q -O- " shquote("https://en.wikipedia.org/w/api.php?action=query&prop=revisions&titles=" urlencodeawk(citeTable[key]["artsecondary"]) "&rvlimit=50&rvslots=main&rvprop=timestamp%7Cuser%7Cids&rvdir=older&format=json&rvcontinue=" urlencodeawk(arcontinue["continue"]) ) if(query_json(sys2var(command), jsona) >= 0) { #debug #awkenough_dump(jsona, "jsona") # jsona["query","pages","1196634","revisions","1","revid"]=7741763 splitja(jsona, arrevid, 5, "revid") # jsona["query","pages","1196634","revisions","1","timestamp"]=2004-11-22T05:21:18Z splitja(jsona, artimestamp, 5, "timestamp") # jsona["continue","rvcontinue"]=20041122055516|7769047 splitja(jsona, arcontinue, 1, "rvcontinue") for(j = 1; j <= length(arrevid); j++) { if(unixTime(artimestamp[j]) <= unixPrimary) { if(unixTime(artimestamp[j]) == unixPrimary && citeTable[key]["artprimary"] == citeTable[key]["artsecondary"]) { # same article same diff get prev if(j == 50) break # size of block requested in API j++ } delete a tup(getwikisource(citeTable[key]["artsecondary"], "dontfollow", "wikipedia.org", "en", arrevid[j]), a) if(a[1] != "REDIRECT" && ! empty(a[1])) { citeTable[key]["artsecondaryrevid"] = arrevid[j] citeTable[key]["artsecondarytimestamp"] = artimestamp[j] citeTable[key]["artsecondarywikitext"] = a[1] cont = 0 break } else { if( empty(a[1]) && empty(a[2]) ) { # revision is empty sendlog(logdir "syslog", wikiname, "Error: empty secondary revision (" citeTable[key]["artsecondary"] "): " artimestamp[j]) } cont = 0 break } } } prevcontinue = arcontinue["continue"] } } } # # Get article title (follow redirects) # function getTitle(key1,key2, dest,a) { citeTable[key1][key2] = "" if(key2 == "artsecondary") { if(match(key1, /^[[][^#]+[^#]/, dest) > 0) { sub(/^[[]{2}/, "", dest[0]) tup(getwikisource(dest[0], "dontfollow", "wikipedia.org", "en"), a) if(a[1] == "REDIRECT") { gsub(/^#REDIRECT[ ]*[[]{2}|[]]$/, "", a[2]) citeTable[key1][key2] = a[2] } else citeTable[key1][key2] = dest[0] } } else if(key2 = "artprimary") { tup(getwikisource(wikiname, "dontfollow", "wikipedia.org", "en"), a) if(a[1] == "REDIRECT") { gsub(/^#REDIRECT [[]{2}|[]]$/, "", a[2]) citeTable[key1][key2] = a[2] } else citeTable[key1][key2] = wikiname } } # # What time and revision was the bug added to the primary article? # populates: # citeTable[key]["artprimaryrevid"] # citeTable[key]["artprimarytimestamp"] # on error they are blank # function getPrimaryRevTime(key, jsona,i,cont,command,arrevid,atimestamp,arcontinue,j,a,prevrevid,prevtimestamp,maxrevs,prevcontinue,jsonin) { citeTable[key]["artprimaryrevid"] = "" citeTable[key]["artprimarytimestamp"] = "" maxrevs = 20 i = 0 cont = 1 while(cont) { i++ if(i > maxrevs) { sendlog(logdir "syslog", wikiname, "Error: Exceeded " maxrevs " API requests in getPrimaryRevTime()") break # sanity break } if(empty(arcontinue["continue"])) command = "wget -q -O- " shquote("https://en.wikipedia.org/w/api.php?action=query&prop=revisions&titles=" urlencodeawk(citeTable[key]["artprimary"]) "&rvlimit=50&rvslots=main&rvprop=timestamp%7Cuser%7Cids&rvdir=older&format=json") else command = "wget -q -O- " shquote("https://en.wikipedia.org/w/api.php?action=query&prop=revisions&titles=" urlencodeawk(citeTable[key]["artprimary"]) "&rvlimit=50&rvslots=main&rvprop=timestamp%7Cuser%7Cids&rvdir=older&format=json&rvcontinue=" urlencodeawk(arcontinue["continue"]) ) jsonin = sys2var(command) if(query_json(jsonin, jsona) >= 0) { #debug #awkenough_dump(jsona, "jsona") # jsona["query","pages","1196634","revisions","1","revid"]=7741763 splitja(jsona, arrevid, 5, "revid") # jsona["query","pages","1196634","revisions","1","timestamp"]=2004-11-22T05:21:18Z splitja(jsona, artimestamp, 5, "timestamp") # jsona["continue","rvcontinue"]=20041122055516|7769047 splitja(jsona, arcontinue, 1, "rvcontinue") if(arcontinue["continue"] == prevcontinue) { # reached last revision j = length(arrevid) delete a tup(getwikisource(citeTable[key]["artprimary"], "dontfollow", "wikipedia.org", "en", arrevid[j]), a) if(a[1] != "REDIRECT" && ! empty(a[1])) { citeTable[key]["artprimaryrevid"] = arrevid[j] citeTable[key]["artprimarytimestamp"] = artimestamp[j] citeTable[key]["artprimarywikitext"] = a[1] cont = 0 break } else { sendlog(logdir "syslog", wikiname, "Error: empty primary revision: " artimestamp[j]) cont = 0 break } } for(j = 1; j <= length(arrevid); j++) { # step through each revision for this batch tup(getwikisource(citeTable[key]["artprimary"], "dontfollow", "wikipedia.org", "en", arrevid[j]), a) if(a[1] != "REDIRECT" && ! empty(a[1])) { # if(countsubstring(a[1], citeTable[key]["decoded"]) == 0) { if(countsubstring(a[1], origTable[key]) == 0) { if(! empty(prevrevid)) { citeTable[key]["artprimaryrevid"] = prevrevid citeTable[key]["artprimarytimestamp"] = prevtimestamp } else { citeTable[key]["artprimaryrevid"] = arrevid[j] citeTable[key]["artprimarytimestamp"] = artimestamp[j] } cont = 0 break } else { prevrevid = arrevid[j] prevtimestamp = artimestamp[j] } } else { if( empty(a[1]) && empty(a[2]) ) { # revision is empty if( citeTable[key]["artprimary"] != citeTable[key]["secondary"] ) { citeTable[key]["artprimaryrevid"] = arrevid[j] citeTable[key]["artprimarytimestamp"] = artimestamp[j] } else sendlog(logdir "syslog", wikiname, "Error: empty primary revision: " artimestamp[j]) cont = 0 break } } } prevcontinue = arcontinue["continue"] } } } # # Given a Wikipedia datestring, return unix-time string (seconds since 1970) # function unixTime(s) { return sys2var("date --date=" shquote(s) " +%s") } # # Return todays date as "August 2019" # function getDateToday() { return sys2var("date +\"%B %Y\"") } # # Determine article date type {{set dmy dates}} etc # imported from medilibrary.nim # function setdatetype(art, reDmy,reMdy,i,a) { reDmy = "[{]{2}" ReSpace "use" ReSpace "dmy" ReSpace "d?a?t?e?s?|[{]{2}" ReSpace "dmy" ReSpace "[|]|[{]{2}" ReSpace "dmy" ReSpace "[}]|[{]{2}" "use[ -]?dmy" reMdy = "[{]{2}" ReSpace "use" ReSpace "mdy" ReSpace "d?a?t?e?s?|[{]{2}" ReSpace "mdy" ReSpace "[|]|[{]{2}" ReSpace "mdy" ReSpace "[}]|[{]{2}" "use[ -]?mdy" for(i = 1; i <= splitn(art, a, i); i++) { if(a[i] ~ reDmy) return "{{dmy}}" if(a[i] ~ reMdy) return "{{mdy}}" } return "" } # # Abort check # function abort(key1, key2) { if( empty( strip(citeTable[key1][key2]) ) ) { if(key2 == "citesecondaryplaintext") { sendlog(logdir "logabort", wikiname, key1 " ---- " citeTable[key1]["citesecondaryplaintext"] " ---- " key2 " missing") } else { sendlog(logdir "logabort", wikiname, key1 " ---- " key2 " missing") } return 1 } return 0 } # # Reverse string # function reverse(s, i,len,a,r) { len = split(s, a, "") for(i = 1; i <= len; i++) r = a[i] r return r } function saveTable() { # Save debuggung info to table.out in data directory printtable = 1 if(printtable) { for(kk in citeTable) { print "\n -------------- \n" >> DataDir "table.out" print "origTable[" kk "] = " origTable[kk] >> DataDir "table.out" for(ll in citeTable[kk]) { if(ll != "artsecondarywikitext") print "citeTable[" kk "][" ll "] = " citeTable[kk][ll] >> DataDir "table.out" else { print "citeTable[" kk "][" ll "] = " length(citeTable[kk][ll]) >> DataDir "table.out" } } } } }
