User:Polygnotus/Scripts/FindDeduplicatable.js
Code that you insert on this page could contain malicious content capable of compromising your account. If you import a script from another page with "importScript", "mw.loader.load", "iusc", or "lusc", take note that this causes you to dynamically load a remote script, which could be changed by others. Editors are responsible for all edits and actions they perform, including by scripts. User scripts are not centrally supported and may malfunction or become inoperable due to software changes. A guide to help you find broken scripts is available. If you are unsure whether code you are adding to this page is safe, you can ask at the appropriate village pump.
This code will be executed when previewing this page.
This code will be executed when previewing this page.
Documentation for this user script can be added at User:Polygnotus/Scripts/FindDeduplicatable.
// <nowiki>
//needed for testing DeduplicateReferences.js
// Wikipedia Duplicate Reference Finder for common.js
// Adds a tool to search for articles with duplicate references that DeduplicateReferences.js can fix
$(document).ready(function() {
// Only run on Wikipedia
if (mw.config.get('wgSiteName') !== 'Wikipedia') return;
// Add portlet link to toolbox
mw.util.addPortletLink(
'p-tb',
'#',
'Find Duplicate Refs',
'find-duplicate-refs',
'Search for articles with duplicate references'
);
// Add the search interface
$('#find-duplicate-refs').click(function(e) {
e.preventDefault();
showDuplicateRefFinder();
});
function showDuplicateRefFinder() {
// Remove existing dialog if present
$('#duplicate-ref-finder').remove();
// Create the search interface
const dialog = $(`
<div id="duplicate-ref-finder" style="
position: fixed;
top: 50px;
right: 20px;
width: 400px;
background: white;
border: 2px solid #0645ad;
border-radius: 5px;
padding: 15px;
box-shadow: 0 4px 8px rgba(0,0,0,0.2);
z-index: 1000;
font-family: sans-serif;
">
<h3 style="margin-top: 0; color: #0645ad;">Duplicate Reference Finder</h3>
<div style="margin-bottom: 10px;">
<label>Search Method:</label><br>
<input type="radio" name="searchMethod" value="random" id="method-random" checked>
<label for="method-random">Random articles</label><br>
<input type="radio" name="searchMethod" value="category" id="method-category">
<label for="method-category">From category</label><br>
<input type="radio" name="searchMethod" value="search" id="method-search">
<label for="method-search">Search term</label>
</div>
<div id="search-input" style="margin-bottom: 10px; display: none;">
<input type="text" id="search-term" placeholder="Enter search term or category" style="width: 100%; padding: 5px;">
</div>
<div style="margin-bottom: 10px;">
<label>Min references to check: </label>
<input type="number" id="min-refs" value="10" min="5" max="100" style="width: 60px;">
</div>
<div style="margin-bottom: 15px;">
<button id="start-search" style="background: #0645ad; color: white; padding: 8px 15px; border: none; border-radius: 3px; cursor: pointer;">Start Search</button>
<button id="stop-search" style="background: #d33; color: white; padding: 8px 15px; border: none; border-radius: 3px; cursor: pointer; margin-left: 5px;" disabled>Stop</button>
<button id="close-finder" style="float: right; background: #666; color: white; padding: 8px 15px; border: none; border-radius: 3px; cursor: pointer;">Close</button>
</div>
<div id="search-status" style="margin-bottom: 10px; font-weight: bold;"></div>
<div id="search-progress" style="margin-bottom: 10px;"></div>
<div id="results" style="max-height: 300px; overflow-y: auto; border: 1px solid #ccc; padding: 10px; background: #f9f9f9;"></div>
</div>
`);
$('body').append(dialog);
// Event handlers
$('input[name="searchMethod"]').change(function() {
if ($(this).val() === 'random') {
$('#search-input').hide();
} else {
$('#search-input').show();
$('#search-term').attr('placeholder',
$(this).val() === 'category' ? 'Enter category name (e.g., "Living people")' : 'Enter search term'
);
}
});
$('#close-finder').click(() => $('#duplicate-ref-finder').remove());
let searchActive = false;
window.currentSearchAborted = false;
$('#start-search').click(async function() {
if (searchActive) return;
searchActive = true;
window.currentSearchAborted = false;
$(this).prop('disabled', true);
$('#stop-search').prop('disabled', false);
$('#results').empty();
$('#search-status').text('Starting search...');
try {
await performSearch();
} catch (error) {
$('#search-status').text('Search error: ' + error.message);
}
searchActive = false;
$('#start-search').prop('disabled', false);
$('#stop-search').prop('disabled', true);
});
$('#stop-search').click(function() {
window.currentSearchAborted = true;
$('#search-status').text('Search stopped by user.');
});
}
async function performSearch() {
const method = $('input[name="searchMethod"]:checked').val();
const minRefs = parseInt($('#min-refs').val()) || 10;
const searchTerm = $('#search-term').val();
let articles = [];
// Get list of articles to check
if (method === 'random') {
articles = await getRandomArticles(50);
} else if (method === 'category' && searchTerm) {
articles = await getCategoryArticles(searchTerm, 50);
} else if (method === 'search' && searchTerm) {
articles = await getSearchResults(searchTerm, 50);
} else {
$('#search-status').text('Please enter a search term or category.');
return;
}
if (articles.length === 0) {
$('#search-status').text('No articles found to check.');
return;
}
$('#search-status').text(`Checking ${articles.length} articles...`);
let checkedCount = 0;
let foundCount = 0;
for (const article of articles) {
if (window.currentSearchAborted) break;
checkedCount++;
$('#search-progress').text(`Checked: ${checkedCount}/${articles.length} | Found: ${foundCount}`);
try {
const duplicateCount = await checkArticleForDuplicates(article.title, minRefs);
if (duplicateCount > 0) {
foundCount++;
addResult(article.title, duplicateCount);
}
} catch (error) {
console.log(`Error checking ${article.title}:`, error);
}
// Small delay to avoid overwhelming the API
await new Promise(resolve => setTimeout(resolve, 200));
}
$('#search-status').text(`Search complete! Checked ${checkedCount} articles, found ${foundCount} with duplicates.`);
}
async function getRandomArticles(count) {
const api = new mw.Api();
const result = await api.get({
action: 'query',
list: 'random',
rnnamespace: 0,
rnlimit: count,
format: 'json'
});
return result.query.random;
}
async function getCategoryArticles(category, count) {
const api = new mw.Api();
category = category.replace(/^Category:/, '');
const result = await api.get({
action: 'query',
list: 'categorymembers',
cmtitle: 'Category:' + category,
cmnamespace: 0,
cmlimit: count,
format: 'json'
});
return result.query.categorymembers;
}
async function getSearchResults(term, count) {
const api = new mw.Api();
const result = await api.get({
action: 'query',
list: 'search',
srsearch: term,
srnamespace: 0,
srlimit: count,
format: 'json'
});
return result.query.search;
}
// Replicate the deduplicator's logic for finding references
function findAllReferences(text) {
const refs = [];
let pos = 0;
while (pos < text.length) {
const refStart = text.indexOf('<ref', pos);
if (refStart === -1) break;
const tagEnd = text.indexOf('>', refStart);
if (tagEnd === -1) break;
const openingTag = text.substring(refStart, tagEnd + 1);
if (openingTag.endsWith('/>')) {
refs.push({text: openingTag, start: refStart, end: tagEnd + 1});
pos = tagEnd + 1;
} else {
const closeStart = text.indexOf('</ref>', tagEnd);
if (closeStart === -1) {
pos = tagEnd + 1;
continue;
}
const fullRef = text.substring(refStart, closeStart + 6);
refs.push({text: fullRef, start: refStart, end: closeStart + 6});
pos = closeStart + 6;
}
}
return refs;
}
// Replicate the deduplicator's normalization logic
function normalizeRef(ref) {
let normalized = ref;
// Remove name attribute
normalized = normalized.replace(/\s*name\s*=\s*(["']?)[^"'\s/>]+(?:\s+[^"'\s/>]+)*\1/i, '');
// Extract content between <ref> tags
const refContentMatch = normalized.match(/<ref[^>]*>([\s\S]*?)<\/ref>/i);
if (!refContentMatch) {
return normalized.replace(/\s+/g, ' ').trim();
}
const refContent = refContentMatch[1];
// For cite templates, normalize parameter order
if (refContent.toLowerCase().includes('{{cite') || refContent.toLowerCase().includes('{{citation')) {
const params = [];
let depth = 0;
let currentParam = '';
for (let i = 0; i < refContent.length; i++) {
const char = refContent[i];
const nextChar = refContent[i + 1] || '';
if ((char === '{' && nextChar === '{') || (char === '[' && nextChar === '[')) {
depth++;
currentParam += char + nextChar;
i++;
} else if ((char === '}' && nextChar === '}') || (char === ']' && nextChar === ']')) {
depth--;
currentParam += char + nextChar;
i++;
if (depth === 0) {
if (currentParam.trim()) {
params.push(currentParam.replace(/\s+/g, ' ').trim());
}
break;
}
} else if (char === '|' && depth === 1) {
if (currentParam.trim()) {
params.push(currentParam.replace(/\s+/g, ' ').trim());
}
currentParam = '';
} else {
currentParam += char;
}
}
if (params.length > 1) {
const templateNameWithBraces = params[0];
const lastParamWithBraces = params[params.length - 1];
const templateName = templateNameWithBraces.replace(/^\{\{/, '');
const lastParam = lastParamWithBraces.replace(/\}\}$/, '');
const middleParams = params.slice(1, -1);
const allParams = [...middleParams, lastParam];
const sortedParams = allParams.sort();
normalized = '<ref>{{' + templateName + '|' + sortedParams.join('|') + '}}</ref>';
} else {
normalized = '<ref>' + refContent.replace(/\s+/g, ' ').trim() + '</ref>';
}
} else {
normalized = '<ref>' + refContent.replace(/\s+/g, ' ').trim() + '</ref>';
}
return normalized;
}
function extractExistingName(ref) {
const nameMatch = ref.match(/name\s*=\s*(["']?)([^"'\s/>]+(?:\s+[^"'\s/>]+)*)\1/i);
return nameMatch ? nameMatch[2] : null;
}
function isSelfClosingNamed(ref) {
return ref.trim().endsWith('/>') && extractExistingName(ref) !== null;
}
async function checkArticleForDuplicates(title, minRefs) {
const api = new mw.Api();
const result = await api.get({
action: 'query',
titles: title,
prop: 'revisions',
rvprop: 'content',
rvslots: 'main',
format: 'json'
});
const pages = result.query.pages;
const pageId = Object.keys(pages)[0];
const page = pages[pageId];
if (!page.revisions || !page.revisions[0]) {
return 0;
}
const content = page.revisions[0].slots.main['*'];
// Find all references using the same logic as deduplicator
const refs = findAllReferences(content);
if (refs.length < minRefs) {
return 0;
}
// Group by normalized content, excluding self-closing named refs
const refGroups = {};
refs.forEach(ref => {
// Skip self-closing named refs (these are just pointers)
if (isSelfClosingNamed(ref.text)) {
return;
}
const normalized = normalizeRef(ref.text);
if (refGroups[normalized]) {
refGroups[normalized].count++;
} else {
refGroups[normalized] = { count: 1 };
}
});
// Count how many groups have duplicates
let duplicateGroupCount = 0;
for (const key in refGroups) {
if (refGroups[key].count > 1) {
duplicateGroupCount++;
}
}
return duplicateGroupCount;
}
function addResult(title, duplicateCount) {
const editUrl = `/w/index.php?title=${encodeURIComponent(title)}&action=edit`;
const viewUrl = `/wiki/${encodeURIComponent(title)}`;
const resultHtml = `
<div style="margin-bottom: 10px; padding: 8px; border: 1px solid #ddd; border-radius: 3px; background: white;">
<strong><a href="${viewUrl}" target="_blank">${mw.html.escape(title)}</a></strong><br>
<span style="color: #d33;">${duplicateCount} duplicate reference group${duplicateCount > 1 ? 's' : ''}</span><br>
<a href="${editUrl}" target="_blank" style="color: #0645ad;">Edit article</a>
</div>
`;
$('#results').append(resultHtml);
const resultsDiv = $('#results')[0];
resultsDiv.scrollTop = resultsDiv.scrollHeight;
}
});
// </nowiki>