User:Polygnotus/Scripts/DeduplicateReferences3.js
Code that you insert on this page could contain malicious content capable of compromising your account. If you import a script from another page with "importScript", "mw.loader.load", "iusc", or "lusc", take note that this causes you to dynamically load a remote script, which could be changed by others. Editors are responsible for all edits and actions they perform, including by scripts. User scripts are not centrally supported and may malfunction or become inoperable due to software changes. A guide to help you find broken scripts is available. If you are unsure whether code you are adding to this page is safe, you can ask at the appropriate village pump.
This code will be executed when previewing this page.
This code will be executed when previewing this page.
Documentation for this user script can be added at User:Polygnotus/Scripts/DeduplicateReferences3.
// <nowiki>
/**
* DeduplicateReferences
* Renames exact-duplicate unnamed <ref> tags so they share a named reference.
* Only runs when editing the current revision of an article (namespace 0),
* or when editing/viewing the designated test page.
*
* [[User:Polygnotus/Scripts/DeduplicateReferences]]
*/
(function () {
'use strict';
// ── Configuration ───────────────────────────────────────────────────────────
const SCRIPT_LINK = '[[User:Polygnotus/Scripts/DeduplicateReferences|DeduplicateReferences]]';
// Change this to your actual test page title (spaces, not underscores)
const TEST_PAGE = 'User:Polygnotus/Scripts/DeduplicateReferencesTest3';
// Hostnames whose slug would produce a useless ref name (e.g. "doi_org").
// When matched, name generation falls through to the next strategy.
const DOMAIN_BLACKLIST = new Set([
'doi.org', 'dx.doi.org', 'jstor.org', 'amazon.com',
'books.google.com', 'web.archive.org', 'archive.org',
'worldcat.org', 'patents.google.com',
]);
// ── Ref parsing ─────────────────────────────────────────────────────────────
//
// Returns an array of token objects, each with:
// { start, end, raw, selfClose }
//
// Positions (start/end) are character offsets into the original string.
// We track positions so replacements can be applied back-to-front without
// recalculating offsets.
function parseRefs(text) {
const tokens = [];
let i = 0;
while (i < text.length) {
const s = text.indexOf('<ref', i);
if (s === -1) break;
// '<ref' must be followed by '>', '/' or whitespace.
// This excludes '<references>' and similar.
const c = text[s + 4];
if (c !== '>' && c !== '/' && !/\s/.test(c)) {
i = s + 4;
continue;
}
const gt = text.indexOf('>', s);
if (gt === -1) { i = s + 1; continue; }
const openTag = text.slice(s, gt + 1);
if (openTag.endsWith('/>')) {
// Self-closing back-reference: <ref name="foo" />
tokens.push({ start: s, end: gt + 1, raw: openTag, selfClose: true });
i = gt + 1;
} else {
// Full ref: find the matching </ref>
const closeStart = text.indexOf('</ref>', gt + 1);
if (closeStart === -1) { i = gt + 1; continue; }
const end = closeStart + 6; // length of '</ref>'
tokens.push({ start: s, end, raw: text.slice(s, end), selfClose: false });
i = end;
}
}
return tokens;
}
// ── Attribute extraction ────────────────────────────────────────────────────
// Generic: extract the value of a named attribute from a ref opening tag.
// Handles double-quoted, single-quoted, and unquoted values.
function attrValue(raw, attrName) {
const re = new RegExp(
`\\b${attrName}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s/>]*))`, 'i'
);
const m = raw.match(re);
if (!m) return null;
const val = (m[1] ?? m[2] ?? m[3] ?? '').trim();
return val === '' ? null : val;
}
const getName = raw => attrValue(raw, 'name');
const getGroup = raw => attrValue(raw, 'group');
// Extract the content between <ref …> and </ref>.
// The outer regex is greedy so it captures everything including nested markup.
const getContent = raw => {
const m = raw.match(/^<ref[^>]*>([\s\S]*)<\/ref>$/i);
return m ? m[1] : '';
};
// ── Normalisation ───────────────────────────────────────────────────────────
//
// Two refs are considered identical when their normalised forms match.
// Normalisation removes the name attribute (so named/unnamed variants of the
// same ref still match) and collapses whitespace.
function normalise(raw) {
return raw
.replace(/\s*\bname\s*=\s*(?:"[^"]*"|'[^']*'|[^\s/>]*)/gi, '')
.replace(/\s+/g, ' ')
.trim();
}
// ── Name generation ─────────────────────────────────────────────────────────
// Convert an arbitrary string to a safe, lowercase ref-name fragment.
function slug(s) {
return s
.normalize('NFD')
.replace(/[\u0300-\u036f]/g, '') // strip combining diacritics
.replace(/[^A-Za-z0-9]+/g, '_')
.replace(/^_+|_+$/g, '')
.toLowerCase()
.slice(0, 40);
}
// Return `base` if unused, otherwise `base_2`, `base_3`, … and register it.
function uniqueName(base, used) {
let name = base;
for (let n = 2; used.has(name); n++) name = `${base}_${n}`;
used.add(name);
return name;
}
// Strategy 1: derive a name from |title= in a cite template.
function nameFromCiteTitle(content) {
const m = content.match(/\|\s*title\s*=\s*([^|{}\n]+)/i);
if (!m) return null;
const words = m[1].replace(/<[^>]*>/g, '').trim().match(/\b[A-Za-z0-9]+\b/g);
return (words && words.length) ? slug(words.slice(0, 3).join('_')) : null;
}
// Strategy 2: derive a name from the first URL in the content.
function nameFromUrl(content) {
const m = content.match(/https?:\/\/([^\s/<>"]+)/i);
if (!m) return null;
let host = m[1].replace(/^www\./, '');
// For archive.org, try to recover the real hostname from the archived URL.
if (/^(web\.)?archive\.org/.test(host)) {
const arc = content.match(
/archive\.org\/web\/\d+\/https?:\/\/(?:www\.)?([^\s/<>"]+)/i
);
if (arc) host = arc[1];
}
const domain = host.split('/')[0];
return DOMAIN_BLACKLIST.has(domain) ? null : slug(domain);
}
// Strategy 3: derive a name from the first few plain-text words.
function nameFromText(content) {
const plain = content
.replace(/<[^>]*>/g, '')
.replace(/\[\[([^|\]]*)[^\]]*\]\]/g, '$1') // strip wikilinks, keep label
.trim();
const words = plain.match(/\b[A-Za-z0-9]+\b/g);
return (words && words.length) ? slug(words.slice(0, 3).join('_')) : null;
}
// Pick the best available name for a ref.
// Priority: cite-template title > URL > plain title= > plain text > 'ref'
function generateName(raw, used) {
const content = getContent(raw);
const hasCite = /\{\{\s*cite|\{\{\s*citation/i.test(content);
if (hasCite) {
const n = nameFromCiteTitle(content);
if (n) return uniqueName(n, used);
}
const n = nameFromUrl(content)
?? nameFromCiteTitle(content)
?? nameFromText(content)
?? 'ref';
return uniqueName(n, used);
}
// ── Core deduplication ──────────────────────────────────────────────────────
/**
* Deduplicate unnamed refs in wikitext.
*
* Rules:
* - Self-closing refs (<ref name="x" />) are never modified.
* - Named refs (<ref name="x">…</ref>) are never modified.
* - Unnamed full refs that appear more than once get a generated name on the
* first occurrence; subsequent occurrences become self-closing back-refs.
* - Group attributes are preserved.
*
* @param {string} text Raw wikitext
* @returns {{ text: string, count: number }}
* `count` = number of duplicate occurrences replaced (not total refs touched)
*/
function deduplicate(text) {
const tokens = parseRefs(text);
// Seed the used-names set with all names already present in the text,
// so auto-generated names never clash with existing ones.
const usedNames = new Set(
tokens.map(t => getName(t.raw)).filter(Boolean)
);
// Group unnamed full refs by normalised content.
// Map: normalisedRef → { first: Token, dupes: Token[] }
const groups = new Map();
for (const tok of tokens) {
if (tok.selfClose) continue; // back-references: skip
if (getName(tok.raw) !== null) continue; // named refs: skip
const key = normalise(tok.raw);
if (!groups.has(key)) {
groups.set(key, { first: tok, dupes: [] });
} else {
groups.get(key).dupes.push(tok);
}
}
// Build a replacement map: start-position → { end, replacementText }
const replacements = new Map();
let count = 0;
for (const { first, dupes } of groups.values()) {
if (dupes.length === 0) continue;
const name = generateName(first.raw, usedNames);
const group = getGroup(first.raw);
const gAttr = group ? ` group="${group}"` : '';
const content = getContent(first.raw);
replacements.set(first.start, {
end: first.end,
text: `<ref name="${name}"${gAttr}>${content}</ref>`,
});
for (const dup of dupes) {
replacements.set(dup.start, {
end: dup.end,
text: `<ref name="${name}"${gAttr} />`,
});
count++;
}
}
if (count === 0) return { text, count: 0 };
// Apply replacements from back to front so earlier positions stay valid.
const result = [...replacements.entries()]
.sort((a, b) => b[0] - a[0])
.reduce(
(t, [start, { end, text: r }]) => t.slice(0, start) + r + t.slice(end),
text
);
return { text: result, count };
}
// ── Edit-page integration ───────────────────────────────────────────────────
function isCurrentRevision() {
const p = new URLSearchParams(window.location.search);
return !p.get('oldid') && !p.get('diff');
}
function runOnPage() {
if (!isCurrentRevision()) return;
const ta = document.getElementById('wpTextbox1');
const sum = document.getElementById('wpSummary');
if (!ta || !ta.value || !sum) return;
const { text, count } = deduplicate(ta.value);
if (count === 0) return;
ta.value = text;
const msg = `[[WP:NAMEDREFS|Deduplicated]] ${count} reference${count !== 1 ? 's' : ''} using ${SCRIPT_LINK}`;
sum.value = sum.value ? `${sum.value} • ${msg}` : msg;
const minor = document.querySelector('#wpMinoredit, input[name="wpMinoredit"]');
if (minor) minor.checked = true;
alert(`Deduplicated ${count} reference${count !== 1 ? 's' : ''}.`);
}
function addButton() {
if (document.getElementById('dedup-ref-btn')) return;
const ta = document.getElementById('wpTextbox1');
const anchor = document.querySelector('.wikiEditor-ui-toolbar') ?? ta?.parentNode;
if (!anchor) return;
const btn = document.createElement('button');
btn.id = 'dedup-ref-btn';
btn.type = 'button';
btn.textContent = 'Deduplicate references';
btn.style.margin = '5px';
if (isCurrentRevision()) {
btn.onclick = runOnPage;
} else {
btn.disabled = true;
btn.title = 'Only available for the current revision.';
}
anchor.appendChild(btn);
}
// Wait until the edit textarea actually has content before running,
// since WikiEditor may inject it asynchronously.
function waitAndRun(fn) {
const ta = document.getElementById('wpTextbox1');
if (ta && ta.value && document.getElementById('wpSummary')) {
fn();
} else {
setTimeout(() => waitAndRun(fn), 150);
}
}
// ── Self-tests ──────────────────────────────────────────────────────────────
//
// These tests run against the deduplicate() function directly (not against any
// page content), so they are stable regardless of what is in the test page.
// The panel is shown when visiting or editing the test page.
const TESTS = [
{
id: 'simple-dup',
desc: 'Two identical unnamed refs: first gets name, second becomes self-closing',
input: 'A<ref>{{cite web|url=https://example.com|title=Foo Bar Baz}}</ref> B<ref>{{cite web|url=https://example.com|title=Foo Bar Baz}}</ref>',
expectCount: 1,
check: out => out.includes('<ref name=') && out.includes('/>'),
},
{
id: 'named-ref-unchanged',
desc: 'Named ref is never modified',
input: 'A<ref name="Smith2020">content</ref> B<ref name="Smith2020" />',
expectCount: 0,
check: out => out === 'A<ref name="Smith2020">content</ref> B<ref name="Smith2020" />',
},
{
id: 'triple-dup',
desc: 'Three identical unnamed refs: 1 named + 2 self-closing, count = 2',
input: 'A<ref>same</ref> B<ref>same</ref> C<ref>same</ref>',
expectCount: 2,
check: out => (out.match(/<ref name=/g) ?? []).length === 3,
},
{
id: 'no-dup',
desc: 'Two different unnamed refs are not touched',
input: 'A<ref>first content</ref> B<ref>second content</ref>',
expectCount: 0,
check: out => !out.includes('<ref name='),
},
{
id: 'group-preserved',
desc: 'group= attribute is preserved on both the named and self-closing ref',
input: 'A<ref group="note">note text</ref> B<ref group="note">note text</ref>',
expectCount: 1,
check: out => (out.match(/group="note"/g) ?? []).length === 2,
},
{
id: 'name-from-cite-title',
desc: 'Name is derived from |title= in a cite template',
input: 'A<ref>{{cite web|title=Hello World Test|url=https://example.com}}</ref>B<ref>{{cite web|title=Hello World Test|url=https://example.com}}</ref>',
expectCount: 1,
check: out => /name="hello_world_test"/.test(out),
},
{
id: 'name-from-url',
desc: 'Name is derived from the URL when there is no cite template',
input: 'A<ref>https://bbc.co.uk/article-123</ref>B<ref>https://bbc.co.uk/article-123</ref>',
expectCount: 1,
check: out => /name="bbc_co_uk"/.test(out),
},
{
id: 'blacklisted-domain-falls-through',
desc: 'Blacklisted domain does not become the name; falls through to text',
input: 'A<ref>{{cite journal|doi=10.1000/xyz|title=Some Journal Article|year=2020}}</ref>B<ref>{{cite journal|doi=10.1000/xyz|title=Some Journal Article|year=2020}}</ref>',
expectCount: 1,
check: out => out.includes('<ref name=') && !/name="doi/.test(out),
},
{
id: 'whitespace-normalised',
desc: 'Refs differing only in whitespace are treated as identical',
input: 'A<ref>content here</ref> B<ref>content here</ref>',
expectCount: 1,
check: out => out.includes('<ref name='),
},
{
id: 'no-name-clash',
desc: 'Generated name does not collide with an existing named ref',
input: 'A<ref name="foo_bar_baz">existing</ref> B<ref>{{cite web|title=Foo Bar Baz}}</ref>C<ref>{{cite web|title=Foo Bar Baz}}</ref>',
expectCount: 1,
// The generated name must differ from the pre-existing 'foo_bar_baz'
check: out => {
const m = out.match(/name="([^"]+)"[^>]*>\{\{cite web\|title=Foo Bar Baz/);
return m !== null && m[1] !== 'foo_bar_baz';
},
},
{
id: 'archive-url-unwrapped',
desc: 'archive.org URL resolves the inner domain for the name',
input: 'A<ref>https://web.archive.org/web/20230101/https://www.bbc.co.uk/news</ref>B<ref>https://web.archive.org/web/20230101/https://www.bbc.co.uk/news</ref>',
expectCount: 1,
check: out => /name="bbc_co_uk"/.test(out),
},
{
id: 'self-closing-not-modified',
desc: 'A lone self-closing ref is never touched',
input: 'A<ref name="foo" /> B<ref name="foo" />',
expectCount: 0,
check: out => out === 'A<ref name="foo" /> B<ref name="foo" />',
},
];
function runTests() {
const results = TESTS.map(tc => {
const { text: out, count } = deduplicate(tc.input);
const ok = count === tc.expectCount && tc.check(out);
return { ...tc, ok, actualCount: count, out };
});
showTestPanel(results);
}
function esc(s) {
return String(s)
.replace(/&/g, '&')
.replace(/</g, '<')
.replace(/>/g, '>');
}
function showTestPanel(results) {
const existing = document.getElementById('dedup-test-panel');
if (existing) existing.remove();
const pass = results.filter(r => r.ok).length;
const fail = results.length - pass;
const border = fail === 0 ? '#388e3c' : '#c62828';
const rows = results.map(r => {
const colour = r.ok ? '#1b5e20' : '#b71c1c';
const extra = !r.ok
? `<br><span style="color:#666;font-size:11px">
count: got ${r.actualCount}, expected ${r.expectCount}<br>
output: ${esc(r.out.slice(0, 140))}
</span>`
: '';
return `<tr style="color:${colour}">
<td style="padding:2px 8px 2px 0;vertical-align:top">${r.ok ? '✓' : '✗'}</td>
<td style="padding:2px 0">${esc(r.desc)}${extra}</td>
</tr>`;
}).join('');
const panel = document.createElement('div');
panel.id = 'dedup-test-panel';
panel.style.cssText = [
'position:fixed', 'top:12px', 'right:12px', 'z-index:9999',
'background:#fafafa', `border:2px solid ${border}`,
'border-radius:6px', 'padding:14px 18px',
'max-width:560px', 'max-height:80vh', 'overflow-y:auto',
'font-family:monospace', 'font-size:12px',
'box-shadow:0 3px 12px rgba(0,0,0,.25)',
].join(';');
panel.innerHTML = `
<strong style="font-size:13px">
DeduplicateReferences — ${pass} / ${results.length} tests passed
</strong>
<button id="dedup-panel-close" type="button"
style="float:right;background:none;border:none;cursor:pointer;font-size:16px;line-height:1;padding:0">✕</button>
<table style="margin-top:10px;border-collapse:collapse;width:100%">${rows}</table>
`;
document.body.appendChild(panel);
document.getElementById('dedup-panel-close').onclick = () => panel.remove();
}
// ── Bootstrap ───────────────────────────────────────────────────────────────
if (typeof mw === 'undefined') {
console.log('DeduplicateReferences: MediaWiki not found, aborting.');
return;
}
const wgAction = mw.config.get('wgAction');
const wgNs = mw.config.get('wgNamespaceNumber');
const wgTitle = mw.config.get('wgPageName');
const testPageName = TEST_PAGE.replace(/ /g, '_');
const isTestPage = (wgTitle === testPageName);
// Show the test panel whenever the test page is loaded (view or edit).
if (isTestPage) {
if (document.readyState === 'complete') {
runTests();
} else {
window.addEventListener('load', runTests);
}
}
// Run deduplication on:
// - article edit pages (namespace 0)
// - the test page in edit mode (so its wikicode can also be tested live)
const shouldRunEdit = wgAction === 'edit' && (wgNs === 0 || isTestPage);
if (shouldRunEdit) {
const setup = () => waitAndRun(() => { runOnPage(); addButton(); });
if (document.readyState === 'complete') {
setup();
} else {
window.addEventListener('load', setup);
}
// Belt-and-suspenders: retry after WikiEditor finishes injecting the toolbar.
setTimeout(addButton, 2000);
}
})();
// </nowiki>