User:Reza1615/BOT/new interwiki.py
< User:Reza1615 | BOT
Before running this code please do localization otherwise it will edit in Farsi :) in your wiki!
- Important Note: Because of changing the API URL you should do this edit in API.py file!
Versions
[edit]- Version 1: developing the bot
- Version 2.00:
- Making Item for multi-language links by one edit
- Adding Namespace Limitation
- Escaping Some API errors
- Solving some minor Bug
- This version is tested.
- Version 2.10:
- solving some minor bugs
- Version 3.00:
- Add possibly to merge single language items and empty the old item and save the list in zzinterwiki_import_merged_need_deleted.txt
- Version 3.50:
- improving the remove interwikis from local wiki (fix some minor bugs)
- remove none-exist interwikis which cause interwiki conflicts
- Version 4.00:
- After deploying new JS tool If we leave NoItewiki and NoItem pages it is much better to not make new Item for them and most of conflicts are caused by single lang items! so this version doesn't make them.
- Version 5.00:
- Add many possibilities such as {{Pages with no Item}} and solving some minor bugs and improving Bot performance
- Version 6.00:
- Solved many minor Bugs which stop bot!
- Version 7.00:
- Solved Bug with wiki voyage and other bug for creating none exist item (now it creates :) )!
- Version 8.00:
- Based on the User:Ladsgroup/Labels, solving the label lang name for some wiki
- Version 9.00:
- Useing merge API and solving some minor bugs
localization part
[edit]- Orgine_lang= your local wiki prefix
- Ignore_Templates=[u'template:speedy delete',u'template:merge'] List of templates (tags) which bot don't add that pages like:
{{Speedy delete}}{{Merge}}you can check your wiki's with their interwiki to Farsi. - Confilict_Template=
u'\n{{Interwiki conflict}}'Conflict template which are listed in Q6099744 - Skip_New=the number of new pages (articles) which should abort to let patrol users to tag
{{Speedy delete}}or{{Merge}}to have less missed language links in wikidata - Local_Remove_IWs_Summary=
u'Bot: removing exist language links in [[wp:wikidata]]: ' - Interwwiki_name=u' Interwiki(s)'
- Orgine_Category_Name= should be the name prefix of Category: in your wiki like رده: for fa.wiki
- Orgine_Template_Name=should be the name prefix of Template: in your wiki like الگو: for fa.wiki
- Orgine_NUM=for languages which had other numbering system like Farsi (u'۰۱۲۳۴۵۶۷۸۹') . if your language uses Numbers like english leave it as u'0123456789'. this numbers used in local wiki's edit summary.
- Local_Remove_IWs_Summary=
u'Bot: removing exist language links in [[wp:wikidata]]: ' - Interwwiki_name=u' Interwiki(s)'
- Remove_IWs_confilict_Summary=u'Bot: Removing Interwiki conflict template'
- Confilict_Template_No_Item=
u'\n{{Pages with no Item}}' - bot_edits=u'Bot: ' #Bot prefix for edit summary
for Toolserver accounts
[edit]in line 47 set your Toolserver's user account (YourUsername)
password_wiki = open("/home/YourUsername/pywikipedia/passfile", 'r')
for enwiki
[edit]Orgine_lang='en'
Ignore_templates=[u'template:speedy delete',u'template:merge']
Confilict_Template=u'\n{{Interwiki conflict}}'
Confilict_Template_No_Item=u'\n{{Pages with no Item}}'
bot_edits=u'Bot: '
Skip_new=200
Editing_Pause=3 # Sleeping time in Second for eding in wikidata
Orgine_Category_Name=u'Category:'
Orgine_Template_Name=u'Template:'
Orgine_NUM=u'0123456789'
Local_Remove_IWs_Summary=u'Bot: removing exist language links in [[wp:wikidata]]: '
Interwwiki_name=u' Interwiki(s)'
Remove_IWs_confilict_Summary=u'Bot: Removing Interwiki conflict template'
code
[edit]#!/usr/bin/python
from __future__ import unicode_literals
# -*- coding: utf-8 -*-
# Reza (User:reza1615)
# Distributed under the terms of the CC-BY-SA 3.0 .
# -*- coding: utf-8 -*-
import wikipedia,pagegenerators,config
import query,time,login,codecs,re,string
import wikidata
from pywikibot import textlib
version=u'9.01'
#-------------------------------localization--------------------
Orgine_lang='fa'
Ignore_Templates=[u'الگو:ویکی‌داده نه',u'الگو:رده بهتر',u'الگو:حذف سریع',u'الگو:حذف زماندار/پیغام',u'الگو:پیشنهاد حذف ۲',u'الگو:ادغام با',u'الگو:ادغام شد در',u'الگو:درخواست ادغام',u'الگو:رده پنهان',u'الگو:Hiddencat',u'الگو:انتقال رده',u'الگو:تغییر مسیر رده',u'الگو:DB',u'الگو:DELETE',u'الگو:Db',u'الگو:Db-Reason',u'الگو:Db-because',u'الگو:Db-reason',u'الگو:Delbecause',u'الگو:Delete because',u'الگو:Delete',u'الگو:Deletebecause',u'الگو:Speedy delete',u'الگو:Speedy',u'الگو:Speedydelete',u'الگو:حذف فوری',u'الگو:حس',u'الگو:کاندید حذف',u'الگو:بدون منبع مدت‌دار',u'الگو:بدون منبع ۲',u'الگو:بدون منبع۲',u'الگو:Prod',u'الگو:نامزد حذف',u'الگو:Proposed deletion',u'الگو:حذف زماندار']
Orgine_Category_Name=u'رده:'
Orgine_Template_Name=u'الگو:'
Orgine_NUM=u'۰۱۲۳۴۵۶۷۸۹'
Confilict_Template=u'\n{{تداخل میان‌ویکی}}'
Confilict_Template_No_Item=u'\n{{صفحه بدون آیتم}}'
bot_edits=u'ربات: '
Skip_new=200
Editing_Pause=0 # Pause in second between edits
Local_Remove_IWs_Summary=u'ربات: حذف میان‌ویکی موجود در [[وپ:ود|ویکی‌داده]]: '
Interwwiki_name=u' میان‌ویکی'
Remove_IWs_confilict_Summary=u'ربات:حذف الگوی تداخل رفع شده'
rep_label= {"no":"nb",
"als":"gsw",
"fiu-vro":"vro",
"bat-smg":"sgs",
"be-x-old":"be-tarask",
"roa-rup":"rup",
"zh-classical":"lzh",
"zh-min-nan":"nan",
"zh-yue":"yue",
"crh":"crh-latn",
"simple":"en"}
#----------------------------------------------------------------
#-------------------Summary which uses in wikidata and they should be english ----
creat_summary="Bot: Import page from {0}wiki".format(Orgine_lang)
update_link_summary="Bot: Update site links from {0}wiki".format(Orgine_lang)
update_link_labels_summary="Bot: Update site links and labels from {0}wiki".format(Orgine_lang)
update_Labels_summary=u'Bot: Update of labels.'
#-------------------------------------------------------------------------------
Orgine_Site=wikipedia.getSite(Orgine_lang,fam='wikipedia')
repo = Orgine_Site.data_repository()
SafeWork=True
mysite=wikipedia.getSite('wikidata','wikidata')
def login_wiki(mode):
if mode==1:
dataSite=wikipedia.getSite('wikidata','wikidata')
if mode==2:
dataSite=wikipedia.getSite(Orgine_lang,'wikipedia')
try:
password_wiki = open("/home/YourUsername/pywikipedia/passfile", 'r')
except:
password_wiki = open(wikipedia.config.datafilepath(config.password_file), 'r')
password_wiki=password_wiki.read().replace('"','').strip()
passwords=password_wiki.split(',')[1].split(')')[0].strip()
usernames=password_wiki.split('(')[1].split(',')[0].strip()
botlog=login.LoginManager(password=passwords,username=usernames,site=dataSite)
botlog.login()
def remove_old_IW (page,item_page):
removed_iw,text,old_text=[],u'',u''
global item_is_updtaed
wikipedia.output(str(item_is_updtaed))
if not item_is_updtaed:
wikipedia.output("\03{lightred}++Because of some problem the item doesn't updated so we need the tereditional interwikis++\03{default}")
return False
My_Template=Confilict_Template
WD_Data = wikipedia.DataPage(item_page)
if not WD_Data.exists():
return True
WD_Dict=WD_Data.get()
WD_Dict= WD_Dict['links']
try:
text=page.get()
except wikipedia.IsRedirectPage:
page = page.getRedirectTarget()
try:
text=page.get()
except:
return True
except:
return True
text=text.replace(u'[[ ',u'[[').replace(u' ]]',u']]').replace(u'[[ ',u'[[').replace(u' ]]',u']]').replace(u"\u200E"+u']]',u']]').replace(u'[['+u"\u200E",u'[[')
text=text.replace(u'Template: ',u'Template:').replace(u'Category: ',u'Category:').replace(u'Wikipedia: ',u'Wikipedia:').replace(u'Portal: ',u'Portal:').replace(u'category: ',u'Category:')
our_en_interwiki=u''
if '[[en:' in text:
our_en_interwiki = u'[[en:'+text.split(u'[[en:')[1].split(u']]')[0].strip()+u']]'
text=text.replace(our_en_interwiki,our_en_interwiki.replace(u'_',u' '))
if u'{{noexternallanglinks}}' in text:
return True
if not text:
save_file(u'[['+page.title()+u']]','error')
return False
text=text.replace(u'\r',u'')
interwikis=textlib.getLanguageLinks(text)
if item_page!=page:
My_Template=Confilict_Template_No_Item
My_Template2=Confilict_Template
else:
My_Template=Confilict_Template
My_Template2=Confilict_Template_No_Item
old_text=text
if interwikis:
for iw in interwikis:
IW_site=interwikis[iw].site().language()
IW_link=interwikis[iw].title()
Lower_IW_link=IW_link[0].lower()+IW_link[1:]
Text_IW_link=u'[['+IW_site+u':'+IW_link+u']]'
Text_IW_link2=u'[['+IW_site+u': '+IW_link+u']]'
Text_IW_link3=u'[['+IW_site+u' :'+IW_link+u']]'
lower_Text_IW_link=u'[['+IW_site+u':'+Lower_IW_link+u']]'
IW_prf=IW_site.replace(u'-',u'_')+u'wiki'
ineterwiki_row_link=IW_link.replace(u'_',u' ').strip()
ineterwiki_row_link_target=redirect_find( ineterwiki_row_link,IW_site).replace(u'_',u' ').strip()
if (IW_prf in WD_Dict) or ((IW_prf in WD_Dict) and (('#' in IW_link) or (u'#' in ineterwiki_row_link_target) or ineterwiki_row_link!=ineterwiki_row_link_target)):
wikipedia.output('\03{lightred}- \03{default}'+Text_IW_link)
text=text.replace(Text_IW_link+u'\n',u'').replace(Text_IW_link,u'').replace(Text_IW_link.replace(u' ',u'_'),u'').replace(Text_IW_link.replace(u'[[',u'[['),u'').replace(Text_IW_link.replace(u']]',u']]'),u'')
text=text.replace(Text_IW_link2+u'\n',u'').replace(Text_IW_link2,u'').replace(Text_IW_link2.replace(u' ',u'_'),u'')
text=text.replace(Text_IW_link3+u'\n',u'').replace(Text_IW_link3,u'').replace(Text_IW_link3.replace(u' ',u'_'),u'')
text=text.replace(our_en_interwiki+u'\n',u'').replace(our_en_interwiki,u'').replace(our_en_interwiki.replace(u' ',u'_'),u'')
text=text.replace(Text_IW_link.replace('Category:',u'category:').replace(' ',u'_')+u'\n',u'')
text=text.replace(lower_Text_IW_link+u'\n',u'').replace(lower_Text_IW_link,u'')
removed_iw.append(IW_site)
else:
if (My_Template.strip() in text) or (My_Template2.strip() in text):
text=text.replace(My_Template.strip()+u'\n',u'').replace(My_Template,u'').replace(My_Template.strip(),u'').replace(My_Template2.strip()+u'\n',u'').replace(My_Template2,u'').replace(My_Template2.strip(),u'')
wikipedia.output('\03{lightred} -'+My_Template+' \03{default}')
text=text.replace(My_Template.strip()+u'\n'+My_Template2.strip(),My_Template.strip()).replace(My_Template2.strip()+u'\n'+My_Template.strip(),My_Template.strip())
page.put(text.strip(),Remove_IWs_confilict_Summary)
return True
interwikis=textlib.getLanguageLinks(text)
if old_text!=text:
if interwikis:
if not My_Template.strip() in text:
if page.namespace()!=10 :
text+=My_Template
else:
if text.find(u'</noinclude>')!=-1:
if string.conut(text,'</noinclude>')<2:
text=text.replace(u'</noinclude>',My_Template+u'\n</noinclude>')
else:
text+=u'<noinclude>'+My_Template+u'\n</noinclude>'
else:
text=u'<noinclude>'+My_Template+u'\n</noinclude>'
wikipedia.output('\03{lightblue} +'+My_Template+' \03{default}')
wikipedia.output(u'+++++++++++++++++++')
try:
text=text.replace(My_Template.strip()+u'\n'+My_Template2.strip(),My_Template.strip()).replace(My_Template2.strip()+u'\n'+My_Template.strip(),My_Template.strip())
page.put(text.strip(),bot_edits+' + '+My_Template)
return True
except wikipedia.LockedPage:
save_file(u'[['+page.title()+u']]','LockedPage')
wikipedia.output(u'Skipping (locked page)')
else:
old_text2=text
text=text.replace(My_Template.strip()+u'\n',u'').replace(My_Template,u'').replace(My_Template.strip(),u'').replace(My_Template2.strip()+u'\n',u'').replace(My_Template2,u'').replace(My_Template2.strip(),u'')
if old_text2!=text:
wikipedia.output('\03{lightred} -'+My_Template+' \03{default}')
text = re.sub('[\r\n]{3,}', "\n\n",text)
text=text.replace(u'<noinclude></noinclude>',u'').replace(u'<noinclude>\n</noinclude>',u'').replace(u'<noinclude>\n\n</noinclude>',u'')
text=text.replace(My_Template.strip()+u'\n'+My_Template2.strip(),My_Template.strip()).replace(My_Template2.strip()+u'\n'+My_Template.strip(),My_Template.strip())
page.put(text.strip(),Remove_IWs_confilict_Summary)
return True
text = re.sub('[\r\n]{3,}', "\n\n",text)
text=text.replace(u'<noinclude></noinclude>',u'').replace(u'<noinclude>\n</noinclude>',u'').replace(u'<noinclude>\n\n</noinclude>',u'')
if removed_iw:
if len(removed_iw)>10:
removed_iw_NUM=str(len(removed_iw))
for i in range(0,10):
removed_iw_NUM = removed_iw_NUM.replace(u'0123456789'[i], Orgine_NUM[i])
removed_links_summary=removed_iw_NUM+Interwwiki_name
else:
removed_links_summary=u', '.join(removed_iw)
if interwikis:
text=text.replace(My_Template.strip()+u'\n'+My_Template2.strip(),My_Template.strip()).replace(My_Template2.strip()+u'\n'+My_Template.strip(),My_Template.strip())
page.put(text.strip(),Local_Remove_IWs_Summary+removed_links_summary+u' + '+My_Template)
else:
try:
text=text.replace(My_Template.strip()+u'\n'+My_Template2.strip(),My_Template.strip()).replace(My_Template2.strip()+u'\n'+My_Template.strip(),My_Template.strip())
page.put(text.strip(),Local_Remove_IWs_Summary+removed_links_summary)
except wikipedia.LockedPage:
save_file(u'[['+page.title()+u']]','LockedPage')
wikipedia.output(u'Skipping(locked page)')
return True
else:
text=text.replace(My_Template.strip()+u'\n'+My_Template2.strip(),My_Template.strip()).replace(My_Template2.strip()+u'\n'+My_Template.strip(),My_Template.strip())
page.put(text.strip(),Remove_IWs_confilict_Summary)
return True
else:
if interwikis:
if not My_Template.strip() in text:
if page.namespace()!=10:
text+=My_Template
else:
if text.find(u'</noinclude>')!=-1:
text=text.replace(u'</noinclude>',My_Template+u'\n</noinclude>')
else:
return False
wikipedia.output('\03{lightblue} +'+My_Template+' \03{default}')
try:
text=text.replace(My_Template.strip()+u'\n'+My_Template2.strip(),My_Template.strip()).replace(My_Template2.strip()+u'\n'+My_Template.strip(),My_Template.strip())
page.put(text.strip(),bot_edits+' + '+My_Template)
return True
except wikipedia.LockedPage:
save_file(u'[['+page.title()+u']]','LockedPage')
wikipedia.output(u'Skipping (locked page)')
return False
def save_file(case,type):
if type=='merge':
file = 'zzinterwiki_import_merged_need_deleted.txt'
elif type=='error':
file = 'zzinterwiki_import_errors.txt'
elif type=='LockedPage':
file = 'zzinterwiki_Locked_Pages.txt'
else:
file = 'zzinterwiki_conflicts.txt'
try:
file_text = codecs.open(file,'r' ,'utf8' )
file_text = file_text.read().strip()
except:
file_text=u''
if not case in file_text:
with codecs.open(file ,mode = 'a',encoding = 'utf8' ) as f:
f.write(u'\n'+case)
def templatequery(pagelink):
temps=[]
pagelink=pagelink.split(u'#')[0].strip()
if pagelink==u'':
return False
pagelink=pagelink.replace(u' ',u'_')
params = {
'action': 'query',
'prop':'templates',
'titles': pagelink,
'redirects': 1,
'tllimit':500,
}
try:
categoryname = query.GetData(params,Orgine_Site)
for item in categoryname[u'query'][u'pages']:
templateha=categoryname[u'query'][u'pages'][item][u'templates']
break
for temp in templateha:
temps.append(temp[u'title'].replace(u'_',u' '))
return temps
except:
return False
def redirect_find( page_link,wiki):
page_link=page_link.replace(u' ',u'_')
site = wikipedia.getSite(wiki.replace(u'_',u'-'))
params = {
'action': 'query',
'redirects':"",
'titles': page_link
}
query_page = query.GetData(params,site)
try:
redirect_link=query_page[u'query'][u'redirects'][0]['to']
try:
hashpart=query_page[u'query'][u'redirects'][0]['tofragment']
return redirect_link+u'#'+hashpart
except:
return redirect_link
except:
if 'missing=""' in str(query_page):
return u''
else:
return page_link.replace(u'_',u' ')
def Check_Page_Exists(page_link, wiki):
page_link=page_link.replace(u' ',u'_')
site = wikipedia.getSite(wiki.replace(u'_',u'-'))
params = {
'action': 'query',
'prop':'info',
'titles': page_link
}
query_page = query.GetData(params,site)
try:
for i in query_page[u'query'][u'pages']:
redirect_link=query_page[u'query'][u'pages'][i]['pageid']
return True# page existed
except:
return False# page not existed
def get_interwikis(link,lang):
Orgine_Site=wikipedia.getSite(lang.replace(u'_',u'-'),fam='wikipedia')
if link.find('#')!=-1:
return False
if link==u'':
return False
link=link.replace(u' ',u'_')
appenddict={}
try:
params = {
'action': 'query',
'prop': 'langlinks',
'titles': link,
'redirects': 1,
'lllimit':500,
}
pagename = query.GetData(params,Orgine_Site)
for item in pagename[u'query'][u'pages']:
case=pagename[u'query'][u'pages'][item][u'langlinks']
for lang in case:
L_lang=lang['lang']
L_link=lang['*']
if not (L_lang in appenddict):
if L_lang=='nb':
L_lang='no'
appenddict[L_lang]=L_link
return appenddict
except:
return appenddict
def check_item(wiki,link):
try:
site=wikipedia.getSite(wiki.replace(u'_',u'-'),fam='wikipedia')
page=wikipedia.Page(site,link)
data=wikipedia.DataPage(page)
items=data.get()
#return False
except wikipedia.NoPage:
return True
except:
wikipedia.output("\03{lightred}Item has been created. Skipping...\03{default}")
return False
def set_lable(data,new_langs,item):
dic_item=data.get()
old=dic_item['links']
changes=False
for cases in new_langs:
if cases=='nbwiki':
cases='nowiki'
dic_item['links'][cases]=new_langs[cases]
if old!=dic_item['links']:
wikipedia.output('added '+cases+'......................')
for langs in dic_item['links']:
if ('voyage' in langs) or ('commons' in langs) or ('source' in langs) or ('book' in langs) or ('quote' in langs) or ('species' in langs) or ('versity' in langs) or ('tionary' in langs) or ('news' in langs):
wikipedia.output("--> \03{lightred}"+langs+"\03{default}\03{lightblue} is passed becuse it dosen't support!\03{default}")
continue
if langs=='nbwiki':
langs='nowiki'
try:
value=dic_item['links'][langs]['name'].strip()
except:
value=dic_item['links'][langs].strip()
lang=langs.replace('wiki','').replace('_','-')
try:
value=unicode(value,'UTF8')
except:
pass
value=value.replace(u"_",u" ")
if lang !='fa':
value = value.split(u'(')[0].strip()
if lang =='es' or lang=='pt' or lang=='pt-br':
value = value.replace(u"Anexo:",u"")
if lang == 'cs':
value = value.replace(u"Príloha:",u"")
if lang == 'de-ch':
value = value.replace(u"ß",u"ss")
if lang in rep_label:
lang=rep_label[lang]
try :
a=dic_item['label'][lang]
except:
item.labels[lang] = value
changes=True
wikipedia.output('\03{lightgreen}for '+value+' added as label of '+lang+'\03{default}')
if changes:
changes=True
else:
wikipedia.output("Doesn't need any update!")
changes=False
return item,changes
def Update_data(data_add,appenddict):
item = wikidata.api.getItemById(data_add.title())
summary=''
confilict={}
new_langs={}
for lang in appenddict:
if lang=='nb':
lang='no'
site_lang=lang
interlangLinks=appenddict[lang]
status=check_item(site_lang,interlangLinks)
if not status:
wikipedia.output(site_lang+' has confilict!')
confilict[site_lang]=interlangLinks
continue
summary=update_link_summary
item.sitelinks[lang+"wiki"] = interlangLinks
new_langs[lang+"wiki"] = interlangLinks
if confilict:
item_confilict=u'* [['+data_add.title()+u']] Confilict > '
for i in confilict:
item_confilict+=u'[[:'+i+u':'+confilict[i]+u'|'+i+u'wiki]]-'
save_file(item_confilict[:-1],'conflict')
if SafeWork:
wikipedia.output('\03{lightred}-->'+data_add.title()+' Passed! because of safe mode and conflict\03{default}')
return False
global item_is_updtaed
if summary:
item,changes=set_lable(data_add,new_langs,item)
if changes:
summary=update_link_labels_summary
try:
wikidata.api.save(item, summary)
wikipedia.output('\03{lightblue}Page '+data_add.title()+' : '+summary+'\03{default}')
return True
except Exception,e:
try:
wikipedia.output('\03{lightred}Page '+data_add.title()+' Passed! error was : '+str(e)+' \03{default}')
item_is_updtaed=False
except:
wikipedia.output('\03{lightred}Page '+data_add.title()+'Passed!\03{default}')
item_is_updtaed=False
return False
def find_diff(my_data,interwiki_links,namespace):
dictionary = my_data.get()
dictionary= dictionary['links']
appenddict={}
for lang in interwiki_links:
if lang=='nb':
lang='no'
L_lang=lang.replace(u'-',u'_')
L_link=interwiki_links[lang].replace(u'category:',u'Category:')
if not (L_lang in appenddict):
if not ((L_lang+'wiki') in dictionary):
appenddict[L_lang]=L_link
wikipedia.output('\03{lightblue}+ '+L_lang +u' > '+L_link+' \03{default}')
if appenddict:
done=Update_data(my_data,appenddict)
if done:
return True
else:
wikipedia.output(str(appenddict))
item = wikidata.api.getItemById(my_data.title())
new_langs={}
item,changes=set_lable(my_data,new_langs,item)
if changes:
summary=update_Labels_summary
wikidata.api.save(item, summary)
wikipedia.output('\03{lightblue}Page '+my_data.title()+' : '+summary+'\03{default}')
return True
return False
def merge_items(a,b):
#https://www.wikidata.org/w/api.php?action=wbmergeitems&fromid=Q42&toid=Q222
print a
print b
params = {
'action': 'wbmergeitems',
'fromid':a,
'toid': b,
}
mysite=wikipedia.getSite('wikidata','wikidata')
#if mysite:
try:
mege_items = query.GetData(params,mysite)
try:
error= mege_items[u'error']
return error
except:
return True
except:
return False
def check_one_lang(my_data,interwiki_links,Orgine_lang):
other_wikis=False
Orgine_title=my_data.title()
item = wikidata.api.getItemById(Orgine_title)
if len(item.sitelinks) == 1 and len(interwiki_links)>1:
for iw in interwiki_links:
if iw==Orgine_lang:
continue
other_wiki_link=redirect_find( interwiki_links[iw],iw)
if not other_wiki_link.strip():
continue
other_wiki_page = wikipedia.Page(iw, other_wiki_link)
other_wiki_data = wikipedia.DataPage(other_wiki_page)
if other_wiki_data.exists():
other_wikis=True
break
if other_wikis:
#----------------------------------------------Merge case---------------
wikipedia.output(u'\03{lightgreen}====Merging Items!====\03{default}')
merge_result=merge_items(Orgine_title,other_wiki_data.title())
if merge_result:
return True
else:
return False
return False
def run(preloadingGen,hasnew,CreatNew,Remove_IW):
if not CreatNew:
Our_Ignore_Templates=[]
else:
Our_Ignore_Templates=Ignore_Templates
if hasnew:
pagecount=0
else:
pagecount=Skip_new+1
for Orgin_page in preloadingGen:
global item_is_updtaed
item_is_updtaed=True
item_page=Orgin_page
pagecount+=1
other_wiki=False
Origin_namespace=Orgin_page.namespace()
if not Origin_namespace in [0,4,10,12,14,100,102]:
continue
Orgine_title=Orgin_page.title()
wikipedia.output(u'Page: -------- '+Orgine_title+u' --------' )
if not Orgin_page.exists():
wikipedia.output(u'Page '+Orgine_title+ u' not existed so it is passed!')
continue
# checking for ignore templates
if Orgine_title:
#try:
do_work=True
page_templates=templatequery(Orgine_title)
if page_templates:
for template in Our_Ignore_Templates:
if template in page_templates:
do_work=False
wikipedia.output(u'\03{lightred}Page '+Orgine_title+u' had {{'+template.replace(Orgine_Template_Name,u'')+'}} so Bot is passed!\03{default}')
break
if not do_work:
continue
#----
interwiki_links=get_interwikis(Orgine_title,Orgine_lang)# get site's interwikis
interwiki_links[Orgine_lang]=Orgine_title
#------------------
my_page = wikipedia.Page(Orgine_Site, Orgine_title)
my_data = wikipedia.DataPage(my_page)
if len(interwiki_links)==1:
Remove_NOIW=True
if my_data.exists():# creating new item
#------------------------------------------------case 0 (NoIW-Item)---------------
wikipedia.output(u'NoIW-Item= '+Orgine_title)
wikipedia.output(u"Doesn't need update!")
continue
else:
#------------------------------------------------case 1 (NoIW-NoItem)---------------
if CreatNew:
if Origin_namespace>0 and Orgine_title.find(u'/')!=-1:
wikipedia.output(u'It is a subpage!')
continue
if pagecount < Skip_new:
wikipedia.output('Page new rank is \03{lightred}'+str(pagecount)+'\03{default} so Bot is passed!')
continue
wikipedia.output(u'NoIW-\03{lightred}NoItem\03{default}= '+Orgine_title)
for i in range(0,2):
try:
my_data.createitem(creat_summary)
except:
if i<1:
login_wiki(1)
time.sleep(1)
if my_data.exists():
break
if my_data.exists():
wikipedia.output(u'Created item= '+creat_summary)
else:# switch to mode 2 (temparay item)
save_file(u'[['+Orgine_title+u']]','error')
wikipedia.output(u'\03{lightred}Item creation had error..!\03{default}')
wikipedia.output(u'Sleeping for '+str(Editing_Pause)+' Seconds ....')
time.sleep(Editing_Pause)
else:
wikipedia.output(u"Page dosen't have itema and interwiki.After new JS tool if we don't make this kind of item is much better!")
continue
else:
Remove_NOIW=True
if my_data.exists():# normal updating to existed item
#------------------------------------------------case 2 (IW-Item) ---------------
wikipedia.output(u'IW-Item= '+Orgine_title)
#if my_data.exists():
try:
done=find_diff(my_data,interwiki_links,Origin_namespace)
if done:
wikipedia.output(u'\03{lightgreen}Item updated!\03{default}')
else:
wikipedia.output(u'\03{lightyellow}Item did not updated...!\03{default}')
done=check_one_lang(my_data,interwiki_links,Orgine_lang)
if not done:
wikipedia.output(u'\03{lightyellow}Item did not merged ..!\03{default}')
#else:
except:
save_file(u'[['+Orgine_title+u']]','error')
wikipedia.output(u'\03{lightyellow}Item did not updated and bot passes..!\03{default}')
else:
for iw in interwiki_links:
other_wiki_link=redirect_find( interwiki_links[iw],iw)
#wikipedia.output(other_wiki_link)
if not other_wiki_link.strip():
continue
other_wiki_Site=wikipedia.getSite(iw.replace(u'_',u'-'),fam='wikipedia')
other_wiki_page = wikipedia.Page(other_wiki_Site, other_wiki_link)
try:
other_wiki_data = wikipedia.DataPage(other_wiki_page)
except:
continue
if other_wiki_data.exists():# creating new item
other_wiki=True
item_page=other_wiki_data
break
if other_wiki:
#----------------------------------------------case 3 (IW-OtherWikiItem)---------------
wikipedia.output(u'IW-\03{lightgreen}OtherWikiItem\03{default}= '+Orgine_title)
wikipedia.output(u'Updating item...')
wikipedia.output(iw+ u':'+other_wiki_link)
try:
done=find_diff(other_wiki_data,interwiki_links,Origin_namespace)
if done:
wikipedia.output(u'\03{lightgreen}Item updated!\03{default}')
else:
done=check_one_lang(other_wiki_data,interwiki_links,Orgine_lang)
if not done:
wikipedia.output(u'\03{lightyellow}Item did not merged ..!\03{default}')
except:
save_file(u'[['+Orgine_title+u']]','error')
wikipedia.output(u'\03{lightyellow}Item did not updated and bot passes..!\03{default}')
else:
#----------------------------------------------case 4 (IW-NoItem)---------------
wikipedia.output(u'IW-\03{lightred}NoItem\03{default}= '+Orgine_title)
sitelinks,labels = [],[]
for iw in interwiki_links:
inter_value=interwiki_links[iw].replace(u' ',u'_')
if iw !='fa':
inter_value = inter_value.split(u'(')[0].strip()
if iw =='es' or iw=='pt' or iw=='pt-br':
inter_value = inter_value.replace(u"Anexo:",u"")
if iw == 'cs':
inter_value = inter_value.replace(u"Príloha:",u"")
if iw == 'de-ch':
inter_value = inter_value.replace(u"ß",u"ss")
iw_l=iw
if iw in rep_label:
iw_l=rep_label[iw]
sitelinks.append({"site": iw+"wiki", "title": interwiki_links[iw]})
labels.append({"language": iw_l, "value": inter_value.replace(u'_',u' ')})
values = {"sitelinks": sitelinks,"labels": labels}
wikipedia.output(u"Creating item for %s" % (Orgin_page.title()))
my_page2 = wikipedia.Page(wikipedia.getSite(iw,fam='wikipedia'), inter_value)
my_data2 = wikipedia.DataPage(my_page2)
try:
my_data2.createitem(value = values, summary = creat_summary)
except:
wikipedia.output(u"\03{lightred}Bot couldn't make an item for "+iw+" : "+inter_value+u" so it is passed!\03{default}")
save_file(u'[['+Orgine_title+u']]','error')
wikipedia.output(u'Sleeping for '+str(Editing_Pause)+' Seconds ....')
time.sleep(Editing_Pause)
#----Removing Iterwikis from Local wiki (it needes to have a flag on local wiki!)
if Remove_IW and Remove_NOIW:
remove_result=remove_old_IW (Orgin_page,item_page)
if not remove_result:
wikipedia.output(u"\03{lightpurple}Bot couldn't remove interwikis from Local wiki!\03{default}")
#except:
# continue
def main():
wikipedia.config.put_throttle = 0
gen=None
wikipedia.put_throttle.setDelay()
hasnew,preloadingGen,CreatNew,Remove_IW=False,False,False,True
PageTitles,namespaces = [],''
genFactory = pagegenerators.GeneratorFactory()
for arg in wikipedia.handleArgs():
if arg.startswith('-newcat'):
arg=arg.replace(':','')
if len(arg) == 7:
genfa = pagegenerators.NewpagesPageGenerator(200, False, Orgine_Site,14)
else:
genfa = pagegenerators.NewpagesPageGenerator(int(arg[7:])+Skip_new, False, Orgine_Site,14)
preloadingGen = pagegenerators.PreloadingGenerator( genfa,60)
hasnew=True
break
elif arg.startswith('-newspace'):
arg=arg.replace(':','')
if len(arg) == 9:
genfa = pagegenerators.NewpagesPageGenerator(2000, False, Orgine_Site,10)
else:
genfa = pagegenerators.NewpagesPageGenerator(200+Skip_new, False, Orgine_Site,int(arg[9:]))
preloadingGen = pagegenerators.PreloadingGenerator( genfa,60)
hasnew=True
break
elif arg.startswith ('-new'):
arg=arg.replace(':','')
if len(arg) == 4:
genfa = pagegenerators.NewpagesPageGenerator(200, False, Orgine_Site,0)
else:
genfa = pagegenerators.NewpagesPageGenerator(int(arg[4:])+Skip_new, False, Orgine_Site,0)
preloadingGen = pagegenerators.PreloadingGenerator( genfa,60)
hasnew=True
break
elif arg.startswith ('-remove'):
Remove_IW=False
elif arg == '-autotitle':
autoTitle = True
elif arg.startswith('-page'):
if len( arg ) == 5:
PageTitles.append( wikipedia.input( u'Which page do you want to chage?' ) )
else:
PageTitles.append( arg[6:] )
break
elif arg.startswith('-namespace:'):
namespaces=int(arg[11:])
elif arg.startswith('-force'):
SafeWork = False
elif arg.startswith('-file'):
textfilename = arg[6:]
if not textfilename:
textfilename = pywikibot.input(
u'Please enter the local file name:')
gen = pagegenerators.TextfilePageGenerator(textfilename,site=Orgine_Site)
elif arg.startswith('-CreatNew'):
CreatNew = True
else:
generator = genFactory.handleArg( arg )
if generator:
gen = generator
if not gen:
wikipedia.stopme()
if PageTitles:
pages = [wikipedia.Page( Orgine_Site,PageTitle ) for PageTitle in PageTitles]
gen = iter( pages )
if namespaces:
gen = pagegenerators.NamespaceFilterPageGenerator( gen,namespaces )
if not preloadingGen:
preloadingGen = pagegenerators.PreloadingGenerator( gen,pageNumber = 60)
run(preloadingGen,hasnew,CreatNew,Remove_IW)
if __name__ == "__main__":
wikipedia.output(u'\03{lightpurple} *******************************\03{default}')
wikipedia.output(u'\03{lightpurple} * Code version is '+version+u' *\03{default}')
wikipedia.output(u'\03{lightpurple} *******************************\03{default}')
login_wiki(1)
login_wiki(2)
item_is_updtaed=True
main()