#!/usr/bin/env python
# -*- coding: utf-8 -*-
import difflib
import simplejson as json # safely retrieve json objects (and correctly handle '/' in article titles)
import pickle # save arrays in files
import re
#import string # string.atoi - variable wait when lagged
import sys # read/write files
import time # what day is it?
import urllib # read/write websites
null = 0
cj = None
ClientCookie = None
cookielib = None
try:
import cookielib
except ImportError:
pass
else:
import urllib2
urlopen = urllib2.urlopen
cj = cookielib.LWPCookieJar()
Request = urllib2.Request
if not cookielib:
try:
import ClientCookie
except ImportError:
import urllib2
urlopen = urllib2.urlopen
Request = urllib2.Request
else:
urlopen = ClientCookie.urlopen
cj = ClientCookie.LWPCookieJar()
Request = ClientCookie.Request
if cj != None:
if cookielib:
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
urllib2.install_opener(opener)
else:
opener = ClientCookie.build_opener(ClientCookie.HTTPCookieProcessor(cj))
ClientCookie.install_opener(opener)
### LOWER-LEVEL URL INTERFACE ###
def act (txdata, url = 'http://en.wikipedia.org/w/api.php', txheaders = {'User-agent' : 'VWBot'}):
while True: # Loop so that it will continue to retry until it connects to the server, handles error occasionally thrown by server
try:
req = Request(url, txdata, txheaders)
handle = urlopen(req)
except IOError, e:
#print 'We failed to open "%s".' % url
#if hasattr(e, 'code'):
# print 'We failed with error code - %s.' % e.code
#elif hasattr(e, 'reason'):
# print "The error object has the following 'reason' attribute :", e.reason
# print "This usually means the server doesn't exist, is down, or we don't have an internet connection."
time.sleep(5)
else:
return handle.read()
# handle.info() returns headers, handle.read() returns the page, handle.geturl() returns the true url of the page fetched (in case urlopen has followed any redirects)
### THIS DOES NOT ACCOUNT FOR QUERY-CONTINUE RESULTS, THESE MUST BE RE-QUERIED LATER
def action (params):
if 'url' in params:
url = params['url']
del params['url']
else: url = 'http://en.wikipedia.org/w/api.php'
while True: # Loop so that it passes all of the errors
params['format'] = 'json'
# An appropriate non-aggressive value is maxlag=5 (5 seconds), used by most of the server-side scripts.
# Higher values mean more aggressive behaviour, lower values are nicer.
#params['maxlag'] = 2 - impractical due to number
params['assert'] = 'bot'
# If we're trying to make an edit, get an edit token first and set the timestamps to recognize an edit conflict.
if params['action'] == 'edit':
page = action({'action': 'query', 'prop': 'info|revisions', 'intoken': 'edit', 'titles': params['title']})
params['token'] = page['query']['pages'][page['query']['pages'].keys()[0]]['edittoken']
params['starttimestamp'] = page['query']['pages'][page['query']['pages'].keys()[0]]['starttimestamp']
if 'revisions' in page['query']['pages'][page['query']['pages'].keys()[0]].keys():
# page exists
params['basetimestamp'] = page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['timestamp']
else:
# page doesn't exist
params['basetimestamp'] = params['starttimestamp']
page = json.loads(act(urllib.urlencode(params), url))
# log reply
file = open(time.strftime('log %Y-%m-%d.txt', time.gmtime()), 'a')
file.write(time.asctime(time.gmtime()) + '\t' + str(page) + '\n\n')
file.close()
# make sure we got a result
if params['action'] in page.keys()[0]:
# if 'continue' in params['action']:
if params['action'] == 'edit': time.sleep(5)
return page
if page['error']['code'] == 'emptynewsection':
return page
# We've lagged: wait the duration of the lag (or a minimum of 5 seconds) and try again
#if page['error']['code'] == 'maxlag':
# time.sleep(max(5,string.atoi(page['error']['info'][page['error']['info'].find(':')+2:page['error']['info'].find('seconds')-1])))
# We've hit an edit conflict or some other unknown error.
time.sleep(5)
#######################
##### @ 00:00 GMT #####
#######################
startTime = time.time()
##### 2-STEP LOGIN #####
def login():
page = action({'action': 'login', 'lgname': foo, 'lgpassword': bar})
page = action({'action': 'login', 'lgname': foo, 'lgpassword': bar, 'lgtoken': page['login']['token']})
if page['login']['result'] == 'Throttled':
time.sleep(page['login']['wait'])
login()
login()
##### TASK 1 #####
# TASK 2 - backlogSCV()
page = action({'action': 'edit', 'bot': 1, 'title': 'Wikipedia:Suspected copyright violations', 'appendtext': time.strftime('\n{{/%Y-%m-%d}}', time.gmtime()), 'section': 0, 'summary': time.strftime('Adding new listing for %-d %B %Y ([[WP:BOT|bot]]) ([[User:VernoWhitney|op]])', time.gmtime())})
##### TASK 3 #####
page = action({'action': 'query', 'list': 'categorymembers', 'cmtitle': 'Category:Articles tagged for copyright problems', 'cmlimit': 'max'})
blankedPages = []
for i in page['query']['categorymembers']:
blankedPages.append(i['title'])
file = open('todayLogCopyvio', 'rb') # pull up the previous run
alreadyBlankedPages = pickle.load(file)
file.close()
file = open('yesterdayLogCopyvio', 'wb') # overwrite yesterday's log with today's now that we have the change in articles
pickle.dump(alreadyBlankedPages, file)
file.close()
file = open('todayLogCopyvio', 'wb') # save log so it can be compared to the next run
pickle.dump(blankedPages, file)
file.close()
newBlankedPages = []
for x in blankedPages:
if x not in alreadyBlankedPages:
newBlankedPages.append(x) # now we have our list to run searches for
for i in newBlankedPages:
if i[:5] == 'File:':
newBlankedPages.remove(i) # also need to report elsewhere - list at [[WP:PUF?]]
##### TASK 5 #####
page = action({'action': 'query', 'list': 'embeddedin', 'eititle': 'Template:Close paraphrasing', 'eilimit': 'max'})
closeParaphrases = []
for i in page['query']['embeddedin']:
closeParaphrases.append(i['title'])
file = open('todayLogCloseParaphrasing', 'rb') # pull up the previous run
oldCloseParaphrases = pickle.load(file)
file.close()
file = open('yesterdayLogCloseParaphrasing', 'wb') # overwrite yesterday's log with today's now that we have the change in articles
pickle.dump(oldCloseParaphrases, file)
file.close()
file = open('todayLogCloseParaphrasing', 'wb') # save log so it can be compared to the next run
pickle.dump(closeParaphrases, file)
file.close()
newCloseParaphrases = []
for x in closeParaphrases:
if x not in oldCloseParaphrases:
newCloseParaphrases.append(x) # now we have our list to run searches for
##### TASK 10 #####
page = action({'action': 'query', 'list': 'embeddedin', 'eititle': 'Template:Copypaste', 'eilimit': 'max'})
copyPastes = []
for i in page['query']['embeddedin']:
copyPastes.append(i['title'])
file = open('todayLogCopypaste', 'rb') # pull up the previous run
oldCopyPastes = pickle.load(file)
file.close()
file = open('yesterdayLogCopypaste', 'wb') # overwrite yesterday's log with today's
pickle.dump(oldCopyPastes, file)
file.close()
file = open('todayLogCopypaste', 'wb') # save the new log so it can be compared to the next run tomorrow
pickle.dump(copyPastes, file)
file.close()
newCopyPastes = []
for x in copyPastes:
if x not in oldCopyPastes:
newCopyPastes.append(x) # now we have our list to run searches for
#######################
##### @ 00:10 GMT #####
#######################
while time.time() - startTime < 600: # no earlier than 00:10 GMT
time.sleep(600 - (time.time() - startTime))
# always update NewListings - this is only needed so Zorglbot doesn't screw up; has no actual effect
page = action({'action': 'edit', 'bot': 1, 'title': 'Wikipedia:Copyright problems/NewListings', 'text': time.strftime('{{Wikipedia:Copyright problems/{{#time:Y F j|-7 day}}}}\n{{Wikipedia:Copyright problems/{{#time:Y F j|-6 day}}}}\n{{Wikipedia:Copyright problems/{{#time:Y F j|-5 day}}}}\n{{Wikipedia:Copyright problems/{{#time:Y F j|-4 day}}}}\n{{Wikipedia:Copyright problems/{{#time:Y F j|-3 day}}}}\n{{Wikipedia:Copyright problems/{{#time:Y F j|-2 day}}}}\n{{Wikipedia:Copyright problems/{{#time:Y F j|-1 day}}}}\n{{Wikipedia:Copyright problems/{{#time:Y F j}}}}<!--\n{{Wikipedia:Copyright problems/%Y %B %-d}}\n', time.gmtime(time.time()-60*60*168)) + time.strftime('{{Wikipedia:Copyright problems/%Y %B %-d}}\n', time.gmtime(time.time()-60*60*144)) + time.strftime('{{Wikipedia:Copyright problems/%Y %B %-d}}\n', time.gmtime(time.time()-60*60*120)) + time.strftime('{{Wikipedia:Copyright problems/%Y %B %-d}}\n', time.gmtime(time.time()-60*60*96)) + time.strftime('{{Wikipedia:Copyright problems/%Y %B %-d}}\n', time.gmtime(time.time()-60*60*72)) + time.strftime('{{Wikipedia:Copyright problems/%Y %B %-d}}\n', time.gmtime(time.time()-60*60*48)) + time.strftime('{{Wikipedia:Copyright problems/%Y %B %-d}}\n', time.gmtime(time.time()-60*60*24)) + time.strftime('{{Wikipedia:Copyright problems/%Y %B %-d}}', time.gmtime()), 'summary': time.strftime('Automatic addition of new listing for %-d %B %Y and archiving of listings older than 7 days ([[WP:BOT|bot]]) ([[User:VernoWhitney|op]])', time.gmtime())})
#######################
##### @ 00:20 GMT #####
#######################
while time.time() - startTime < 1200: # no earlier than 00:20 GMT
time.sleep(1200 - (time.time() - startTime))
##### TASK 3 #####
p3 = re.compile('<!-- This is Cppage. Comment used by User:DumbBOT, do not remove or change -->')
p4 = re.compile('====.*====')
page = action({'action': 'query', 'prop': 'revisions', 'rvprop': 'content', 'titles': time.strftime('Wikipedia:Copyright problems/%Y %B %-d', time.gmtime()), 'rvlimit': 1})
# group new page creation AND old page archival
if 'missing' in page['query']['pages'][page['query']['pages'].keys()[0]]:
# CREATE AND POPULATE "BOT: Automatic creation of new daily page for copyright problems"
page = action({'action': 'edit', 'bot': 1, 'title': time.strftime('Wikipedia:Copyright problems/%Y %B %-d', time.gmtime()), 'text': '{{subst:Cppage}}\n<!-- Add new listings at the bottom of the list with the following format:\n\n* {{subst:article-cv|ArticleName}} from [http://www.WhereItWasCopiedFrom.com]. ~~~~\n\n-->\n', 'summary': 'Automatic creation of new daily page for copyright problems including automated findings ([[WP:BOT|bot]]) ([[User:VernoWhitney|op]])'})
page = action({'action': 'query', 'prop': 'revisions', 'rvprop': 'content', 'titles': 'Wikipedia:Copyright problems', 'rvlimit': 1})
newtext = page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['*'].replace('\n\n===New listings===', time.strftime('\n{{Wikipedia:Copyright problems/%Y %B %-d}}\n\n===New listings===', time.gmtime(time.time()-60*60*192)))
page = action({'action': 'edit', 'bot': 1, 'title': 'Wikipedia:Copyright problems', 'text': newtext.encode('utf-8'), 'summary': 'Automatic archiving of listings older than 7 days ([[WP:BOT|bot]]) ([[User:VernoWhitney|op]])'})
elif not re.search(p3, page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['*']):
# POPULATE "adding CorenSearchBot findings"
page = action({'action': 'edit', 'bot': 1, 'title': time.strftime('Wikipedia:Copyright problems/%Y %B %-d', time.gmtime()), 'text': page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['*'].replace(re.search(p4, page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['*']).group(),'{{subst:Cppage}}'), 'summary': 'Adding automated findings ([[WP:BOT|bot]]) ([[User:VernoWhitney|op]])'})
##### TASKS 3, 5, 7 and 10 #####
def isAlreadyListed(title):
page = action({'action': 'query', 'list': 'backlinks', 'bltitle': title.encode('utf-8'), 'bllimit': 'max', 'blfilterredir': 'redirects'})
page['query']['backlinks'].append({'title': title})
for i in page['query']['backlinks']:
page = action({'action': 'query', 'list': 'backlinks', 'bltitle': i['title'].encode('utf-8'), 'bllimit': 'max', 'blnamespace': '4'})
for j in page['query']['backlinks']:
if 'Wikipedia:Copyright problems' == j['title'] or 'Wikipedia:Suspected copyright violations' == j['title'] or 'Wikipedia:Copyright problems/NewListings' == j['title']:
return True
return False
# replace NewListings check with one for each of the 8 always-listed days ???
def shouldBeRelisted(title):
page = action({'action': 'query', 'list': 'backlinks', 'bltitle': title.encode('utf-8'), 'bllimit': 'max', 'blfilterredir': 'redirects'})
page['query']['backlinks'].append({'title': title})
wasListed = False
isListed = False
for i in page['query']['backlinks']:
page = action({'action': 'query', 'list': 'backlinks', 'bltitle': i['title'].encode('utf-8'), 'bllimit': 'max', 'blnamespace': '4'})
for j in page['query']['backlinks']:
if 'Wikipedia:Suspected copyright violations/' in j['title'] or 'Wikipedia:Copyright problems/' in j['title']:
wasListed = True
if 'Wikipedia:Copyright problems' == j['title'] or 'Wikipedia:Suspected copyright violations' == j['title'] or 'Wikipedia:Copyright problems/NewListings' == j['title']:
isListed = True
if wasListed and not isListed: return True
return False
# replace NewListings check with one for each of the 8 always-listed days ???
addtext = ''
p0 = re.compile('{{Close paraphras.*?}}', re.IGNORECASE | re.DOTALL)
p1 = re.compile('{{Close paraphras.*?source.*?}}', re.IGNORECASE | re.DOTALL) # gets {{Close paraphrase}} and {{Close paraphrasing}}
p1a = re.compile('\|\W*free\W*=\W*yes', re.IGNORECASE | re.DOTALL) # is source free?
for i in newCloseParaphrases:
if not isAlreadyListed(i):
page = action({'action': 'query', 'prop': 'revisions', 'rvprop': 'content', 'titles': i.encode('utf-8'), 'rvlimit': 1})
if 'missing' not in page['query']['pages'][page['query']['pages'].keys()[0]]:
pageSource = page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['*']
if re.search(p0, pageSource): # could be tag removed before it's analyzed
temp = re.search(p0, pageSource).group()
tag = re.search(p1, temp)
if not re.search(p1a, temp): # only list at WP:CP if non-free
if tag:
if '|' in tag.group()[tag.group().find('source') + tag.group()[tag.group().find('source'):].find('='):]:
addtext += '* {{subst:article-cv|:' + i + '}} Close paraphrase of ' + tag.group()[tag.group().find('source') +\
tag.group()[tag.group().find('source'):].find('=') + 1:tag.group().find('source') + tag.group()[tag.group().find('source'):].find('=') +\
tag.group()[tag.group().find('source') + tag.group()[tag.group().find('source'):].find('='):].find('|')].strip() + '. ~~~~\n'
else:
addtext += '* {{subst:article-cv|:' + i + '}} Close paraphrase of ' +\
tag.group()[tag.group().find('source') + tag.group()[tag.group().find('source'):].find('=') + 1:-2].strip() + '. ~~~~\n'
else: addtext += '* {{subst:article-cv|:' + i + '}} Close paraphrase. ~~~~\n'
moretext = ''
p2 = re.compile('{{Copyviocore.*?}}', re.IGNORECASE | re.DOTALL)
for i in newBlankedPages:
if not isAlreadyListed(i):
page = action({'action': 'query', 'prop': 'revisions', 'rvprop': 'content', 'titles': i.encode('utf-8'), 'rvlimit': 1})
if 'missing' not in page['query']['pages'][page['query']['pages'].keys()[0]]:
pageSource = page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['*']
tag = re.search(p2, pageSource)
if tag:
if '|' in tag.group()[tag.group().find('url') + tag.group()[tag.group().find('url'):].find('='):]:
moretext += '* {{subst:article-cv|:' + i + '}} from ' + tag.group()[tag.group().find('url') +\
tag.group()[tag.group().find('url'):].find('=') + 1:tag.group().find('url') + tag.group()[tag.group().find('url'):].find('=') +\
tag.group()[tag.group().find('url') + tag.group()[tag.group().find('url'):].find('='):].find('|')].strip() + '. Nomination completed by ~~~~\n'
else:
moretext += '* {{subst:article-cv|:' + i + '}} from ' +\
tag.group()[tag.group().find('url') + tag.group()[tag.group().find('source'):].find('=') + 1:-2].strip() + '. Nomination completed by ~~~~\n'
else: moretext += '* {{subst:article-cv|:' + i + '}} Nomination completed by ~~~~\n'
CopyPasteText = ''
p5 = re.compile('{{Copy.?past.*?}}|{{Copy\s*\|.*?}}|{{Copy\s*}}', re.IGNORECASE | re.DOTALL)
p6 = re.compile('{{Copy.?past.*?url.*?}}|{{Copy\s*\|.*?url.*?}}', re.IGNORECASE | re.DOTALL)
for i in newCopyPastes:
if not isAlreadyListed(i):
page = action({'action': 'query', 'prop': 'revisions', 'rvprop': 'content', 'titles': i.encode('utf-8'), 'rvlimit': 1})
if 'missing' not in page['query']['pages'][page['query']['pages'].keys()[0]]:
pageSource = page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['*']
if re.search(p5, pageSource): # could be tag removed before it's analyzed
temp = re.search(p5, pageSource).group()
tag = re.search(p6, temp)
if tag:
if '|' in tag.group()[tag.group().find('url') + tag.group()[tag.group().find('url'):].find('='):]:
CopyPasteText += '* {{subst:article-cv|:' + i + '}} Copied and pasted from ' + tag.group()[tag.group().find('url') +\
tag.group()[tag.group().find('url'):].find('=') + 1:tag.group().find('url') + tag.group()[tag.group().find('url'):].find('=') +\
tag.group()[tag.group().find('url') + tag.group()[tag.group().find('url'):].find('='):].find('|')].strip() + '. ~~~~\n'
else:
CopyPasteText += '* {{subst:article-cv|:' + i + '}} Copied and pasted from ' +\
tag.group()[tag.group().find('url') + tag.group()[tag.group().find('url'):].find('=') + 1:-2].strip() + '. ~~~~\n'
else: CopyPasteText += '* {{subst:article-cv|:' + i + '}} Copied and pasted. ~~~~\n'
### NOW FOR THE RELISTINGS ###
evenmoretext = ''
for i in blankedPages:
if i in alreadyBlankedPages and shouldBeRelisted(i): # need to check alreadyBlankedPages as there is a delay between transclusion and backlinks
page = action({'action': 'query', 'prop': 'revisions', 'rvprop': 'content', 'titles': i.encode('utf-8'), 'rvlimit': 1})
if 'missing' not in page['query']['pages'][page['query']['pages'].keys()[0]]:
pageSource = page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['*']
tag = re.search(p2, pageSource)
if tag:
if '|' in tag.group()[tag.group().find('url') + tag.group()[tag.group().find('url'):].find('='):]:
evenmoretext += '* {{subst:article-cv|:' + i + '}} from ' + tag.group()[tag.group().find('url') +\
tag.group()[tag.group().find('url'):].find('=') + 1:tag.group().find('url') + tag.group()[tag.group().find('url'):].find('=') +\
tag.group()[tag.group().find('url') + tag.group()[tag.group().find('url'):].find('='):].find('|')].strip() + '. Relisting. ~~~~\n'
else:
evenmoretext += '* {{subst:article-cv|:' + i + '}} from ' +\
tag.group()[tag.group().find('url') + tag.group()[tag.group().find('source'):].find('=') + 1:-2].strip() + '. Relisting. ~~~~\n'
else: evenmoretext += '* {{subst:article-cv|:' + i + '}} Relisting. ~~~~\n'
for i in copyPastes:
if i in oldCopyPastes and shouldBeRelisted(i):
page = action({'action': 'query', 'prop': 'revisions', 'rvprop': 'content', 'titles': i.encode('utf-8'), 'rvlimit': 1})
if 'missing' not in page['query']['pages'][page['query']['pages'].keys()[0]]:
pageSource = page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['*']
temp = re.search(p5, pageSource).group()
tag = re.search(p6, temp)
if tag:
if '|' in tag.group()[tag.group().find('url') + tag.group()[tag.group().find('url'):].find('='):]:
CopyPasteText += '* {{subst:article-cv|:' + i + '}} Copied and pasted from ' + tag.group()[tag.group().find('url') +\
tag.group()[tag.group().find('url'):].find('=') + 1:tag.group().find('url') + tag.group()[tag.group().find('url'):].find('=') +\
tag.group()[tag.group().find('url') + tag.group()[tag.group().find('url'):].find('='):].find('|')].strip() + '. Relisting. ~~~~\n'
else:
evenmoretext += '* {{subst:article-cv|:' + i + '}} Copied and pasted from ' +\
tag.group()[tag.group().find('url') + tag.group()[tag.group().find('url'):].find('=') + 1:-2].strip() + '. Relisting. ~~~~\n'
else: evenmoretext += '* {{subst:article-cv|:' + i + '}} Copied and pasted. Relisting. ~~~~\n'
for i in closeParaphrases:
if i in oldCloseParaphrases and shouldBeRelisted(i): # need to check alreadyBlankedPages as there is a delay between transclusion and backlinks
page = action({'action': 'query', 'prop': 'revisions', 'rvprop': 'content', 'titles': i.encode('utf-8'), 'rvlimit': 1})
if 'missing' not in page['query']['pages'][page['query']['pages'].keys()[0]]:
pageSource = page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['*']
temp = re.search(p0, pageSource).group()
tag = re.search(p1, temp)
if not re.search(p1a, temp): # only list at WP:CP if non-free
if tag:
if '|' in tag.group()[tag.group().find('source') + tag.group()[tag.group().find('source'):].find('='):]:
evenmoretext += '* {{subst:article-cv|:' + i + '}} Close paraphrase of ' + tag.group()[tag.group().find('source') +\
tag.group()[tag.group().find('source'):].find('=') + 1:tag.group().find('source') + tag.group()[tag.group().find('source'):].find('=') +\
tag.group()[tag.group().find('source') + tag.group()[tag.group().find('source'):].find('='):].find('|')].strip() + '. Relisting. ~~~~\n'
else:
evenmoretext += '* {{subst:article-cv|:' + i + '}} Close paraphrase of ' +\
tag.group()[tag.group().find('source') + tag.group()[tag.group().find('source'):].find('=') + 1:-2].strip() + '. Relisting. ~~~~\n'
else: evenmoretext += '* {{subst:article-cv|:' + i + '}} Close paraphrase. Relisting. ~~~~\n'
#addtext should be CloseParaphraseText
#moretext should be CopyvioText
#evenmoretext should be RelistText
editsum = ''
if len(addtext) + len(moretext) + len(evenmoretext) + len(CopyPasteText):
if len(addtext):
if len(moretext):
if len(evenmoretext):
if len(CopyPasteText): editsum = 'Adding incomplete nominations, copy/pastes, close paraphrases and relisting overlooked pages'
else: editsum = 'Adding incomplete nominations, close paraphrases and relisting overlooked pages'
elif len(CopyPasteText): editsum = 'Adding incomplete nominations, copy/pastes and close paraphrases'
else: editsum = 'Adding incomplete nominations and close paraphrases'
elif len(evenmoretext):
if len(CopyPasteText): editsum = 'Adding copy/pastes, close paraphrases and relisting overlooked pages'
else: editsum = 'Adding close paraphrases and relisting overlooked pages'
elif len(CopyPasteText): editsum = 'Adding copy/pastes and close paraphrases'
else: editsum = 'Adding close paraphrases'
elif len(moretext):
if len(evenmoretext):
if len(CopyPasteText): editsum = 'Adding incomplete nominations, copy/pastes and relisting overlooked pages'
else: editsum = 'Adding incomplete nominations and relisting overlooked pages'
elif len(CopyPasteText): editsum = 'Adding incomplete nominations and copy/pastes'
else: editsum = 'Adding incomplete nominations'
elif len(evenmoretext):
if len(CopyPasteText): editsum = 'Adding copy/pastes and relisting overlooked pages'
else: editsum = 'Relisting overlooked pages'
else: editsum = 'Adding copy/pastes'
if len(editsum):
page = action({'action': 'edit', 'bot': 1, 'title': time.strftime('Wikipedia:Copyright problems/%Y %B %-d', time.gmtime(time.time()-60*60*24)), 'appendtext': (u'\n' + moretext + CopyPasteText + addtext + evenmoretext).encode('utf-8'), 'section': 2, 'summary': editsum + ' ([[WP:BOT|bot]]) ([[User:VernoWhitney|op]])'})
############################
##### USERSPACE TRIALS #####
############################
##### TASK 4: notify authors that their pages have been blanked (by {{subst:copyvio}}) in case they aren't notified by the taggers, so that the pages don't get relisted for an extra week without any action being taken on them #####
def doNotify(title):
page = action({'action': 'query', 'list': 'backlinks', 'bltitle': title.encode('utf-8'), 'bllimit': 'max', 'prop': 'revisions|info', 'rvprop': 'timestamp|user', 'rvdir': 'newer', 'titles': title.encode('utf-8'), 'rvlimit': 1, 'blredirect': 1}) # get backlinks and creation time/user as well as info to determine if it's deleted
if 'missing' in page['query']['pages'][page['query']['pages'].keys()[0]]:
return "'''Do Nothing''' Article has been deleted."
for i in page['query']['backlinks']: # check for CCIs
if i['title'][:47] == 'Wikipedia:Contributor copyright investigations/':
return "'''Do Nothing''' [[" + i['title'] + '|CCI]]'
elif i['title'][:14] == 'Wikipedia:CCI/':
return "'''Do Nothing''' [[" + i['title'] + '|CCI]]'
if 'redirlinks' in i:
for j in i['redirlinks']:
if j['title'][:47] == 'Wikipedia:Contributor copyright investigations/':
return "'''Do Nothing''' [[" + j['title'] + '|CCI]]'
elif j['title'][:14] == 'Wikipedia:CCI/':
return "'''Do Nothing''' [[" + j['title'] + '|CCI]]'
for i in page['query']['backlinks']: # parse talk pages to see if already notified
if i['title'][:10] == 'User talk:':
page2 = action({'action': 'parse', 'page': i['title'], 'prop': 'sections'})
for j in page2['parse']['sections']:
if j['line'] == 'Copyright problem: ' + title: # need to see if it matches a redirect title too... :(
return "'''Do Nothing''' " + i['title'][10:] + ' already notified'
page = action({'action': 'query', 'prop': 'categories', 'clcategories': 'Category:Items pending OTRS confirmation of permission|Category:Wikipedia pages with unconfirmed permission received by OTRS|Category:Wikipedia files with unconfirmed permission received by OTRS|Category:Items with OTRS permission confirmed', 'titles': 'Talk:'+title.encode('utf-8')})
if 'categories' in page['query']['pages'][page['query']['pages'].keys()[0]]:
return "'''Do Nothing''' OTRS tag"
page = action({'action': 'query', 'prop': 'revisions', 'rvprop': 'ids|user', 'titles': title.encode('utf-8'), 'rvlimit': 'max'})
articleRevisionIDs = []
for i in page['query']['pages'][page['query']['pages'].keys()[0]]['revisions']:
articleRevisionIDs.append(i['revid'])
revisionMatch = []
latest = ''
for i in articleRevisionIDs:
page = action({'action': 'query', 'prop': 'revisions', 'rvstartid': i, 'rvprop': 'content|user|timestamp', 'titles': title.encode('utf-8'), 'rvlimit': 1})
if i == articleRevisionIDs[0]: # maybe ???
tagger = page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['user'] # maybe ???
tagtime = page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['timestamp'] # maybe ??
if '*' in page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0].keys(): # ignore deleted revisions
if latest == '': latest = page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['*']
if '{{Copyviocore' in page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['*']:
tagger = page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['user']
tagtime = page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['timestamp']
revisionMatch.append(difflib.SequenceMatcher(None, latest[latest.find('<!-- Do not use the "Copyviocore" template directly; the above line is generated by "subst:Copyvio|url" -->\n')+108:latest.find('</div>')], page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['*']).ratio())
diffRevisionMatch = []
for i in range(len(revisionMatch)):
if i < len(revisionMatch)-1: diffRevisionMatch.append(round(revisionMatch[i]-revisionMatch[i+1], 6))
else: diffRevisionMatch.append(round(revisionMatch[i], 6))
page = action({'action': 'query', 'prop': 'revisions', 'rvprop': 'user', 'titles': title.encode('utf-8'), 'rvlimit': 1, 'rvstartid': articleRevisionIDs[[i for i, x in enumerate(diffRevisionMatch) if x == max(diffRevisionMatch)][0]]})
contributor = page['query']['pages'][page['query']['pages'].keys()[0]]['revisions'][0]['user']
# CHECK FOR CUSTOM NOTIFICATION
#tagger at User talk:contributor > tagtime
page = action({'action': 'query', 'prop': 'revisions', 'rvprop': 'user', 'titles': 'User talk:' + contributor.encode('utf-8'), 'rvend': tagtime, 'rvlimit': 'max'})
if 'revisions' in page['query']['pages'][page['query']['pages'].keys()[0]]:
for i in page['query']['pages'][page['query']['pages'].keys()[0]]['revisions']:
if i['user'] == tagger: return "'''Do Nothing''' " + contributor + ' was left a custom notification'
#contributor at Talk:Article/Temp page > tagtime
page = action({'action': 'query', 'prop': 'revisions', 'rvprop': 'user', 'titles': 'Talk:' + title.encode('utf-8') + '/Temp', 'rvend': tagtime, 'rvlimit': 'max'})
if 'revisions' in page['query']['pages'][page['query']['pages'].keys()[0]]:
for i in page['query']['pages'][page['query']['pages'].keys()[0]]['revisions']:
if i['user'] == contributor: return "'''Do Nothing''' " + contributor + ' created the temporary page'
#contributor at Talk:Article > tagtime
page = action({'action': 'query', 'prop': 'revisions', 'rvprop': 'user', 'titles': 'Talk:' + title.encode('utf-8'), 'rvend': tagtime, 'rvlimit': 'max'})
if 'revisions' in page['query']['pages'][page['query']['pages'].keys()[0]]:
for i in page['query']['pages'][page['query']['pages'].keys()[0]]['revisions']:
if i['user'] == contributor: return "'''Do Nothing''' " + contributor + ' edited the article talk page after it was tagged'
#contributor at Article > tagtime
page = action({'action': 'query', 'prop': 'revisions', 'rvprop': 'user', 'titles': title.encode('utf-8'), 'rvend': tagtime, 'rvlimit': 'max'})
if 'revisions' in page['query']['pages'][page['query']['pages'].keys()[0]]:
for i in page['query']['pages'][page['query']['pages'].keys()[0]]['revisions']:
if i['user'] == contributor: return "'''Do Nothing''' " + contributor + ' edited the article after it was tagged'
return "'''Notify contributor''': """ + contributor + ' - tagged by ' + tagger
#narrowing with 'blnamespace': '3|4' breaks the blredirect parameter
# BETTER BUGFIX - try narrowed backlinks, then get list of redirects ONLY, then get backlinks for each redirect
# look for 'Copyright problem: <title or redirect>'
# list of all blanked pages
article = ''
for i in newBlankedPages:
article += '*[[:' + i + ']] - ' + doNotify(i) + '\n'
page = action({'action': 'edit', 'bot': 1, 'title': 'User:VWBot/Trial', 'text': (article + '\n').encode('utf-8'), 'section': 'new', 'summary': time.strftime('== %-d %B %Y ==', time.gmtime())})
##### TASK 6: flag when a contributor gets a CorenSearchBot/VWBot notice if he has had a significant amount before #####
# CSBot's user talk contribs from 00:00:00 to 23:59:59 the previous day
page = action({'action': 'query', 'list': 'usercontribs', 'ucuser': 'CorenSearchBot', 'uclimit': 'max', 'ucstart': time.strftime('%Y-%m-%dT23:59:59Z', time.gmtime(time.time()-60*60*24)), 'ucend': time.strftime('%Y-%m-%dT00:00:00Z', time.gmtime(time.time()-60*60*24)), 'ucnamespace': '3'})
users = {}
for i in page['query']['usercontribs']:
users[i['title']] = []
# VWBot's user talk contribs from 00:00:00 to 23:59:59 the previous day
page = action({'action': 'query', 'list': 'usercontribs', 'ucuser': 'VWBot', 'uclimit': 'max', 'ucstart': time.strftime('%Y-%m-%dT23:59:59Z', time.gmtime(time.time()-60*60*24)), 'ucend': time.strftime('%Y-%m-%dT00:00:00Z', time.gmtime(time.time()-60*60*24)), 'ucnamespace': '3'})
for i in page['query']['usercontribs']:
users[i['title']] = []
for i in ['Merovingian', u'Leszek Jańczuk', 'Ganeshbot', 'Starzynka', 'Ser Amantio di Nicolao', 'Kumioko', 'Packerfansam', 'Alan Liefting']:
try: del users['User talk:' + i]
except: pass
for user in users.keys():
# only checks last 5,000 edits
page = action({'action': 'query', 'prop': 'revisions', 'rvprop': 'comment|timestamp|user', 'titles': user.encode('utf-8'), 'rvlimit': 'max'})
for i in page['query']['pages'][page['query']['pages'].keys()[0]]['revisions']:
if 'user' in i: # needed because RevDelete can return edits with no user field...apparently
if i['user'] == 'VWBot' or i['user'] == 'CorenSearchBot':
users[user].append([i['comment'][i['comment'].find('on')+3:], time.strftime('%Y %B %-d', time.strptime(i['timestamp'],'%Y-%m-%dT%H:%M:%SZ'))])
addition = u''
for user in users.keys():
if len(users[user]) > 4:
addition += '\n==== ' + str(len(users[user])) + ': {{User|1=' + user[10:] + '}} ====\n{{Collapse top|Tagged articles}}\n'
for i in users[user]:
addition += '* {{subst:article-cv|' + i[0] + '}} created on ' + i[1] + '\n'
addition += '{{Collapse bottom}}\n'
if len(addition):
page = action({'action': 'edit', 'bot': 1, 'title': 'User:VWBot/Trial', 'appendtext': (u'\n\n=== Task 6 ===' + addition).encode('utf-8'), 'summary': 'Listing users who have had multiple articles tagged by CorenSearchBot/VWBot ([[WP:BOT|bot]]) ([[User:VernoWhitney|op]])'})
