Jump to content

User:Polygnotus/Scripts/DeduplicateReferences2.js

From Wikipedia, the free encyclopedia
Note: After saving, you have to bypass your browser's cache to see the changes. Google Chrome, Firefox, Microsoft Edge and Safari: Hold down the ⇧ Shift key and click the Reload toolbar button. For details and instructions about other browsers, see Wikipedia:Bypass your cache.
/*
<ref>{{cite web|title=x|url=y}}</ref>
<ref>{{cite web|title=x    |url=y}}</ref>
<ref>{{cite web|url=y|title=x}}</ref>
*/

// <nowiki>
// Only exact duplicates
// Tries to come up with a name for the reference
// Only runs when editing the current version of the article

// Debug configuration
const DEBUG = true; // Set to false to disable debug logging
const debug = (...args) => {
  if (DEBUG) {
    console.log('[DeduplicateReferences]', ...args);
  }
};

// Function to normalize diacritics for comparison and ref names
function normalizeDiacritics(str) {
  return str.normalize('NFD').replace(/[\u0300-\u036f]/g, '');
}

// Function to deduplicate references in Wikipedia articles
function deduplicateReferences() {
  debug('Starting deduplication process...');
  
  // Get the edit textarea and summary input
  const editTextarea = document.getElementById('wpTextbox1');
  const summaryInput = document.getElementById('wpSummary');
  
  if (!editTextarea || !summaryInput) {
    debug('Edit textarea or summary input not found');
    return;
  }

  let content = editTextarea.value;
  debug('Content length:', content.length);
  
  // Function to find all references in text
  function findAllReferences(text) {
    const refs = [];
    let pos = 0;
    
    while (pos < text.length) {
      const refStart = text.indexOf('<ref', pos);
      if (refStart === -1) break;
      
      // Find the end of the opening tag
      const tagEnd = text.indexOf('>', refStart);
      if (tagEnd === -1) break;
      
      const openingTag = text.substring(refStart, tagEnd + 1);
      
      // Check if it's self-closing
      if (openingTag.endsWith('/>')) {
        refs.push({text: openingTag, start: refStart, end: tagEnd + 1});
        pos = tagEnd + 1;
      } else {
        // Find the closing </ref>
        const closeStart = text.indexOf('</ref>', tagEnd);
        if (closeStart === -1) {
          pos = tagEnd + 1;
          continue;
        }
        
        const fullRef = text.substring(refStart, closeStart + 6);
        refs.push({text: fullRef, start: refStart, end: closeStart + 6});
        pos = closeStart + 6;
      }
    }
    
    return refs;
  }
  
  // Object to store unnamed references only
  const unnamedRefs = {};
  
  // Set to store all used reference names (normalized)
  const usedNames = new Set();
  
  // Blacklist of reference names to ignore
  const blacklist = [
    "doi_org",
    "jstor_org",
    "amazon_com",
    "books_google_com",
    "web_archive_org",
    "worldcat_org",
    "dx_doi_org",
    "patents_google_com",
    "cite_journal",
    "cite_book",
    "cite_web",
    "cite_news",
    "cite_magazine",
    "cite_newspaper",
    "cite_thesis",
    "cite_conference",
    "cite_encyclopedia",
    "cite_album_notes",
    "cite_comic",
    "cite_court",
    "cite_act",
    "cite_episode",
    "cite_mailing_list",
    "cite_map",
    "cite_newsgroup",
    "cite_patent",
    "cite_press_release",
    "cite_report",
    "cite_video_game",
    "citation"
  ];
  
  // Function to extract domain name from URL
  function extractDomain(url) {
    try {
      let domain = new URL(url).hostname;
      domain = domain.replace(/^www\./, '');
      return domain === 'archive.org' ? extractDomain(url.split('archive.org/web/')[1]) : domain;
    } catch (e) {
      return null;
    }
  }
  
  // Function to extract title from cite templates
  function extractTitleFromCiteTemplate(ref) {
    // Look for title parameter in cite templates
    const titleMatch = ref.match(/\|\s*title\s*=\s*([^|{}]+?)(?:\s*\||$)/i);
    if (titleMatch) {
      let title = titleMatch[1].trim();
      // Remove any remaining markup
      title = title.replace(/<[^>]*>/g, '').replace(/\[\[([^\]]*)\]\]/g, '$1');
      // Extract first 3 words
      const words = title.match(/\b[A-Za-z0-9]+\b/g);
      if (words && words.length > 0) {
        return normalizeDiacritics(words.slice(0, 3).join('').toLowerCase());
      }
    }
    return null;
  }

  // Function to generate a unique name for the reference
  function generateUniqueName(ref) {
    // Check if this is a cite template and try to extract title (case insensitive)
    if (ref.toLowerCase().includes('{{cite') || ref.toLowerCase().includes('{{citation')) {
      const titleName = extractTitleFromCiteTemplate(ref);
      if (titleName) {
        let uniqueName = titleName;
        let counter = 1;
        while (usedNames.has(uniqueName)) {
          uniqueName = `${titleName}_${counter}`;
          counter++;
        }
        usedNames.add(uniqueName);
        return uniqueName;
      }
    }
    
    // Try to extract meaningful text from the reference
    const textContent = ref.replace(/<[^>]*>/g, '').trim();
    
    // Look for URLs first
    const urlMatch = ref.match(/https?:\/\/[^\s<>"]+/i);
    if (urlMatch) {
      const domain = extractDomain(urlMatch[0]);
      if (domain && !blacklist.includes(domain.replace(/\./g, '_'))) {
        let baseName = normalizeDiacritics(domain.replace(/\./g, '_'));
        let uniqueName = baseName;
        let counter = 1;
        while (usedNames.has(uniqueName)) {
          uniqueName = `${baseName}_${counter}`;
          counter++;
        }
        usedNames.add(uniqueName);
        return uniqueName;
      }
    }
    
    // If no URL, try to create name from content
    if (textContent) {
      // Extract first 3 meaningful words
      const words = textContent.match(/\b[A-Za-z0-9]+\b/g);
      if (words && words.length > 0) {
        let baseName = normalizeDiacritics(words.slice(0, 3).join('_').toLowerCase());
        let uniqueName = baseName;
        let counter = 1;
        while (usedNames.has(uniqueName)) {
          uniqueName = `${baseName}_${counter}`;
          counter++;
        }
        usedNames.add(uniqueName);
        return uniqueName;
      }
    }
    
    // Fallback to generic name
    let baseName = 'ref';
    let uniqueName = baseName;
    let counter = 1;
    while (usedNames.has(uniqueName)) {
      uniqueName = `${baseName}_${counter}`;
      counter++;
    }
    usedNames.add(uniqueName);
    return uniqueName;
  }
  
  // Function to extract existing name from a reference - only looks in opening tag
  function extractExistingName(ref) {
    // Only search within the opening <ref...> tag
    const openingTagMatch = ref.match(/<ref[^>]*>/i);
    if (!openingTagMatch) return null;
    
    const openingTag = openingTagMatch[0];
    
    // Look for name attribute with quotes
    const quotedNameMatch = openingTag.match(/\bname\s*=\s*["']([^"']+)["']/i);
    if (quotedNameMatch) {
      return quotedNameMatch[1];
    }
    
    // Look for name attribute without quotes (must end at space, / or >)
    const unquotedNameMatch = openingTag.match(/\bname\s*=\s*([^\s"'/>]+)/i);
    if (unquotedNameMatch) {
      return unquotedNameMatch[1];
    }
    
    return null;
  }
  
  // Function to create a reference tag
  function createRefTag(name, content = null) {
    if (content) {
      return `<ref name="${name}">${content}</ref>`;
    } else {
      return `<ref name="${name}" />`;
    }
  }
  
  // Function to normalize reference content for comparison
  function normalizeRef(ref) {
    let normalized = ref;
    
    // Remove name attribute
    normalized = normalized.replace(/\s*name\s*=\s*(["']?)[^"'\s/>]+(?:\s+[^"'\s/>]+)*\1/i, '');
    
    // Extract just the content between <ref> tags
    const refContentMatch = normalized.match(/<ref[^>]*>([\s\S]*?)<\/ref>/i);
    if (!refContentMatch) {
      // For self-closing refs or malformed refs, just normalize whitespace and diacritics
      return normalizeDiacritics(normalized.replace(/\s+/g, ' ').trim());
    }
    
    const refContent = refContentMatch[1];
    
    // For cite templates, normalize parameter order and whitespace
    if (refContent.toLowerCase().includes('{{cite') || refContent.toLowerCase().includes('{{citation')) {
      // Split into parameters while respecting nested templates/links
      const params = [];
      let depth = 0;
      let currentParam = '';
      
      for (let i = 0; i < refContent.length; i++) {
        const char = refContent[i];
        const nextChar = refContent[i + 1] || '';
        
        if ((char === '{' && nextChar === '{') || (char === '[' && nextChar === '[')) {
          depth++;
          currentParam += char + nextChar;
          i++; // skip next char
        } else if ((char === '}' && nextChar === '}') || (char === ']' && nextChar === ']')) {
          depth--;
          currentParam += char + nextChar;
          i++; // skip next char
          // If we've closed the main template, stop
          if (depth === 0) {
            if (currentParam.trim()) {
              params.push(currentParam.replace(/\s+/g, ' ').trim());
            }
            break;
          }
        } else if (char === '|' && depth === 1) {
          if (currentParam.trim()) {
            // Trim whitespace around = signs and at parameter boundaries
            let param = currentParam.replace(/\s+/g, ' ').trim();
            // Also normalize whitespace around the = sign specifically
            param = param.replace(/\s*=\s*/g, '=');
            params.push(param);
          }
          currentParam = '';
        } else {
          currentParam += char;
        }
      }
      
      // Sort all params except the first (template name with opening {{) and last (which has closing }})
      if (params.length > 1) {
        const templateNameWithBraces = params[0]; // e.g., "{{cite web"
        const lastParamWithBraces = params[params.length - 1]; // e.g., "url=y}}"
        
        // Extract template name without opening braces
        const templateName = templateNameWithBraces.replace(/^\{\{/, '').trim();
        
        // Extract last param without closing braces and trim it
        const lastParam = lastParamWithBraces.replace(/\}\}$/, '').replace(/\s+/g, ' ').trim().replace(/\s*=\s*/g, '=');
        
        // Get middle params (if any) - already normalized above
        const middleParams = params.slice(1, -1);
        
        // Combine all params except template name, sort them, then reconstruct
        const allParams = [...middleParams, lastParam];
        const sortedParams = allParams.sort();
        
        normalized = '<ref>{{' + templateName + '|' + sortedParams.join('|') + '}}</ref>';
      } else {
        normalized = '<ref>' + refContent.replace(/\s+/g, ' ').trim() + '</ref>';
      }
    } else {
      // For non-template refs, just normalize whitespace
      normalized = '<ref>' + refContent.replace(/\s+/g, ' ').trim() + '</ref>';
    }
    
    // Normalize diacritics in the final result
    return normalizeDiacritics(normalized);
  }
  
  debug('Starting first pass - collecting unnamed references...');
  
  // First pass: collect all unnamed full references and track used names
  const matches = findAllReferences(content);
  debug('Found', matches.length, 'reference tags');
  
  matches.forEach((match, index) => {
    const existingName = extractExistingName(match.text);
    
    // Track all used names (normalized)
    if (existingName) {
      const normalizedName = normalizeDiacritics(existingName);
      usedNames.add(normalizedName);
      debug('Found existing name:', existingName, '(normalized:', normalizedName + ')');
    }
    
    // Only process unnamed full references (not self-closing, no name attribute)
    if (!existingName && !match.text.trim().endsWith('/>')) {
      const normalizedRef = normalizeRef(match.text);
      
      if (unnamedRefs[normalizedRef]) {
        unnamedRefs[normalizedRef].count++;
        unnamedRefs[normalizedRef].instances.push(match);
      } else {
        unnamedRefs[normalizedRef] = { 
          count: 1, 
          firstOccurrence: match,
          instances: [match]
        };
      }
    }
  });
  
  debug('Unnamed reference groups:', Object.keys(unnamedRefs).length);
  debug('Reference analysis:', Object.keys(unnamedRefs).map(key => ({
    ref: key.substring(0, 50) + '...',
    count: unnamedRefs[key].count
  })));
  
  // Find duplicates (only unnamed refs that appear more than once)
  const duplicates = Object.keys(unnamedRefs).filter(key => 
    unnamedRefs[key].count > 1
  );
  debug('Found', duplicates.length, 'duplicate unnamed reference groups');
  
  // Second pass: replace duplicates with named references
  let deduplicatedCount = 0;
  
  debug('Starting second pass - replacing duplicates...');
  
  // Collect all replacements to make (sorted by position, in reverse order)
  const replacements = [];
  
  // Process each duplicate group
  duplicates.forEach(normalizedRef => {
    const refInfo = unnamedRefs[normalizedRef];
    
    // Generate a unique name for this group
    const generatedName = generateUniqueName(refInfo.firstOccurrence.text);
    
    if (generatedName && !blacklist.includes(generatedName)) {
      debug('Generated name for duplicate group:', generatedName);
      
      // Extract content from the first occurrence
      const contentMatch = refInfo.firstOccurrence.text.match(/<ref[^>]*>([\s\S]*?)<\/ref>/i);
      
      if (contentMatch) {
        const namedFullRef = createRefTag(generatedName, contentMatch[1]);
        
        // Replace first occurrence with named version
        replacements.push({
          start: refInfo.firstOccurrence.start,
          end: refInfo.firstOccurrence.end,
          replacement: namedFullRef
        });
        debug('Will replace first occurrence with named version');
        
        // Replace all subsequent occurrences with short refs
        for (let i = 1; i < refInfo.instances.length; i++) {
          replacements.push({
            start: refInfo.instances[i].start,
            end: refInfo.instances[i].end,
            replacement: createRefTag(generatedName)
          });
          deduplicatedCount++;
          debug('Will replace duplicate occurrence', i, 'with short ref');
        }
      }
    }
  });
  
  // Sort replacements by position (reverse order, so we can replace from end to start)
  replacements.sort((a, b) => b.start - a.start);
  
  // Apply all replacements
  replacements.forEach(repl => {
    content = content.substring(0, repl.start) + repl.replacement + content.substring(repl.end);
  });
  
  debug('Deduplication complete. Count:', deduplicatedCount);
  
  // Update the textarea with the deduplicated content
  if (deduplicatedCount > 0) {
    editTextarea.value = content;
    
    // Add edit summary
    let currentSummary = summaryInput.value;
    let deduplicationSummary = `[[WP:NAMEDREFS|Deduplicated]] ${deduplicatedCount} reference${deduplicatedCount > 1 ? 's' : ''} using [[User:Polygnotus/Scripts/DeduplicateReferences|DeduplicateReferences]]`;
    summaryInput.value = currentSummary ? `${currentSummary} • ${deduplicationSummary}` : deduplicationSummary;
    
    // Check minor edit if available
    const minorEditCheckbox = document.querySelector('#wpMinoredit, input[name="wpMinoredit"]');
    if (minorEditCheckbox) {
      minorEditCheckbox.checked = true;
    }
    
    debug('Successfully deduplicated', deduplicatedCount, 'references');
    alert(`Successfully deduplicated ${deduplicatedCount} reference${deduplicatedCount > 1 ? 's' : ''}!`);
  } else {
    debug('No duplicates found to deduplicate');
    alert('No unnamed duplicate references found to deduplicate.');
  }
}

// Function to check if we're editing the current version (not an old revision)
function isEditingCurrentVersion() {
  // Check if there's an oldid parameter in the URL
  const urlParams = new URLSearchParams(window.location.search);
  const oldid = urlParams.get('oldid');
  
  // If there's an oldid parameter, we're editing an old revision
  if (oldid) {
    debug('Editing old revision (oldid=' + oldid + '), skipping deduplication');
    return false;
  }
  
  // Check if we're in diff view
  const diff = urlParams.get('diff');
  if (diff) {
    debug('In diff view, skipping deduplication');
    return false;
  }
  
  // Check for section editing of old revisions
  const section = urlParams.get('section');
  if (section && oldid) {
    debug('Editing section of old revision, skipping deduplication');
    return false;
  }
  
  // Additional check: look for revision warning messages
  const revisionWarning = document.querySelector('.mw-revision-warning, .mw-editnotice-base');
  if (revisionWarning && revisionWarning.textContent.toLowerCase().includes('old revision')) {
    debug('Old revision warning detected, skipping deduplication');
    return false;
  }
  
  debug('Editing current version - deduplication allowed');
  return true;
}

// Function to check if the edit textarea is ready
function isEditTextareaReady() {
  const editTextarea = document.getElementById('wpTextbox1');
  const summaryInput = document.getElementById('wpSummary');
  return editTextarea && editTextarea.value && summaryInput;
}

// Function to run deduplication when everything is ready
function runDeduplicationWhenReady() {
  debug('Checking if ready...');
  if (isEditTextareaReady()) {
    debug('Ready! Checking if editing current version...');
    if (isEditingCurrentVersion()) {
      debug('Editing current version - running deduplication...');
      deduplicateReferences();
    } else {
      debug('Not editing current version - skipping deduplication');
    }
  } else {
    debug('Not ready yet, retrying...');
    setTimeout(runDeduplicationWhenReady, 100);
  }
}

// Add a button to manually trigger deduplication
function addDeduplicationButton() {
  const toolbar = document.querySelector('.wikiEditor-ui-toolbar') || 
                  document.querySelector('#wpTextbox1').parentNode;
  
  if (toolbar && !document.getElementById('dedupe-button')) {
    const button = document.createElement('button');
    button.id = 'dedupe-button';
    button.type = 'button';
    button.textContent = 'Deduplicate References';
    button.style.margin = '5px';
    
    // Only enable button if editing current version
    if (isEditingCurrentVersion()) {
      button.onclick = deduplicateReferences;
      debug('Added deduplication button (enabled)');
    } else {
      button.disabled = true;
      button.title = 'Deduplication only works when editing the current version of an article';
      button.onclick = () => alert('Deduplication only works when editing the current version of an article, not old revisions.');
      debug('Added deduplication button (disabled - old revision)');
    }
    
    toolbar.appendChild(button);
  }
}

// Run the deduplication when the edit page is fully loaded
if (typeof mw !== 'undefined') {
  const action = mw.config.get('wgAction');
  const namespace = mw.config.get('wgNamespaceNumber');
  
  if ((action === 'edit' || action === 'submit') && namespace === 0) {
    debug('Article edit page detected, checking version and setting up deduplication...');
    
    if (document.readyState === 'complete') {
      runDeduplicationWhenReady();
      addDeduplicationButton();
    } else {
      window.addEventListener('load', () => {
        runDeduplicationWhenReady();
        addDeduplicationButton();
      });
    }
  } else {
    debug('Not on article edit page - action:', action, 'namespace:', namespace);
  }
} else {
  debug('MediaWiki not available');
}

// Also add the button when the page is ready (only for article namespace)
setTimeout(() => {
  if (typeof mw !== 'undefined') {
    const action = mw.config.get('wgAction');
    const namespace = mw.config.get('wgNamespaceNumber');
    
    if (action === 'edit' && namespace === 0) {
      addDeduplicationButton();
    }
  }
}, 2000);

// </nowiki>
Retrieved from "https://en.wikipedia.org/w/index.php?title=User:Polygnotus/Scripts/DeduplicateReferences2.js&oldid=1333578305"