// Wikipedia Category Items Copier - Fixed Redirect Filtering
const API_DELAY = 500; // Delay between API requests in milliseconds
const MAX_RETRIES = 3; // Maximum number of retries for failed requests
// Only run on Wikipedia category pages
if (window.location.href.includes('/wiki/Category:')) {
// Extract the category name from the URL
const categoryName = decodeURIComponent(window.location.pathname.split('/Category:')[1]);
console.log("Category Name:", categoryName);
// Create a container for our buttons
const container = document.createElement('div');
container.style.padding = '10px';
container.style.margin = '10px 0';
container.style.backgroundColor = '#f8f9fa';
container.style.border = '1px solid #a2a9b1';
container.style.borderRadius = '3px';
// Helper function to create tooltip
function addTooltip(element, text) {
element.title = text;
element.style.position = 'relative';
}
// Create the "Copy Items" button
const copyItemsBtn = document.createElement('button');
copyItemsBtn.textContent = 'Copy items';
copyItemsBtn.style.marginRight = '10px';
copyItemsBtn.style.padding = '8px 12px';
copyItemsBtn.style.cursor = 'pointer';
addTooltip(copyItemsBtn, 'Copy all items in this category. Not recursive.');
// Create the "Copy All Items" button
const copyAllItemsBtn = document.createElement('button');
copyAllItemsBtn.textContent = 'Copy items recursively';
copyAllItemsBtn.style.marginRight = '10px';
copyAllItemsBtn.style.padding = '8px 12px';
copyAllItemsBtn.style.cursor = 'pointer';
addTooltip(copyAllItemsBtn, 'Copy all items in this category AND all items in its subcategories.');
// Create the "Copy Subcats from this Category" button
const copyDirectSubcatsBtn = document.createElement('button');
copyDirectSubcatsBtn.textContent = 'Copy subcats';
copyDirectSubcatsBtn.style.marginRight = '10px';
copyDirectSubcatsBtn.style.padding = '8px 12px';
copyDirectSubcatsBtn.style.cursor = 'pointer';
addTooltip(copyDirectSubcatsBtn, 'Copy all subcategories of this category. Not recursive.');
// Create the "Copy Subcategories" button
const copySubcatsBtn = document.createElement('button');
copySubcatsBtn.textContent = 'Copy subcategories recursively';
copySubcatsBtn.style.marginRight = '10px';
copySubcatsBtn.style.padding = '8px 12px';
copySubcatsBtn.style.cursor = 'pointer';
addTooltip(copySubcatsBtn, 'Copy all subcategories of this category and its subcategories.');
// Create the "Copy Both" button
const copyBothBtn = document.createElement('button');
copyBothBtn.textContent = 'Copy both';
copyBothBtn.style.marginRight = '10px';
copyBothBtn.style.padding = '8px 12px';
copyBothBtn.style.cursor = 'pointer';
addTooltip(copyBothBtn, 'Copy all items and subcategories from this category. Not recursive.');
// Create the "Copy Both Recursively" button
const copyBothRecursiveBtn = document.createElement('button');
copyBothRecursiveBtn.textContent = 'Copy both recursively';
copyBothRecursiveBtn.style.marginRight = '10px';
copyBothRecursiveBtn.style.padding = '8px 12px';
copyBothRecursiveBtn.style.cursor = 'pointer';
addTooltip(copyBothRecursiveBtn, 'Copy all items and subcategories from this category and all its subcategories.');
// Add checkbox for URL export
const urlCheckbox = document.createElement('input');
urlCheckbox.type = 'checkbox';
urlCheckbox.id = 'includeUrls';
urlCheckbox.style.marginLeft = '15px';
const urlLabel = document.createElement('label');
urlLabel.htmlFor = 'includeUrls';
urlLabel.textContent = 'Whole URLs';
urlLabel.style.marginLeft = '5px';
addTooltip(urlLabel, 'Include full Wikipedia URLs for each item in the export');
// Create status text
const statusText = document.createElement('div');
statusText.style.marginTop = '10px';
statusText.style.color = '#555';
// Add buttons to container in the requested order
container.appendChild(copyItemsBtn);
container.appendChild(copyAllItemsBtn);
container.appendChild(copyDirectSubcatsBtn);
container.appendChild(copySubcatsBtn);
container.appendChild(copyBothBtn);
container.appendChild(copyBothRecursiveBtn);
container.appendChild(urlCheckbox);
container.appendChild(urlLabel);
container.appendChild(statusText);
// Insert container after the page title
const pageTitleHeading = document.querySelector('.mw-first-heading');
if (pageTitleHeading) {
pageTitleHeading.parentNode.insertBefore(container, pageTitleHeading.nextSibling);
} else {
document.querySelector('#content').prepend(container);
}
// Global visited set to prevent visiting any page more than once across all operations
const globalVisited = new Set();
// Function to format items with URLs if requested
function formatItems(items, includeUrls) {
if (!includeUrls) {
return items.join('\n');
}
// When URLs are requested, return ONLY the URLs, not the article names
return items.map(item => {
const encodedTitle = encodeURIComponent(item.replace(/ /g, '_'));
return `https://en.wikipedia.org/wiki/${encodedTitle}`;
}).join('\n');
}
// Function that creates a download link as an alternative to clipboard
function offerTextAsDownload(text, filename) {
// Create blob from text
const blob = new Blob([text], {type: 'text/plain'});
const url = URL.createObjectURL(blob);
// Create download link
const downloadLink = document.createElement('a');
downloadLink.href = url;
downloadLink.download = filename || 'wikipedia-category-items.txt';
downloadLink.textContent = `Download ${filename || 'items'} as text file`;
downloadLink.style.display = 'block';
downloadLink.style.marginTop = '10px';
// Add to status container
statusText.appendChild(downloadLink);
return true;
}
// Function to copy text to clipboard or offer download if copying fails
function copyToClipboardOrDownload(text, categoryName) {
return new Promise((resolve) => {
// Try to copy to clipboard first
tryClipboardCopy(text).then(success => {
if (success) {
resolve(true);
} else {
// If clipboard fails, offer download instead
const filename = `${categoryName.replace(/[^a-z0-9]/gi, '_')}-items.txt`;
// Clear the status text completely and show only the clipboard failure message
statusText.innerHTML = `<p>Clipboard access failed. Click the link below to download items:</p>`;
offerTextAsDownload(text, filename);
resolve(false);
}
});
});
}
// Try multiple clipboard methods
function tryClipboardCopy(text) {
return new Promise((resolve) => {
// First try the modern Clipboard API
if (navigator.clipboard && navigator.clipboard.writeText) {
navigator.clipboard.writeText(text)
.then(() => resolve(true))
.catch(() => {
// If Clipboard API fails, try execCommand
try {
const textarea = document.createElement('textarea');
textarea.value = text;
// Position off-screen but available
textarea.style.position = 'fixed';
textarea.style.left = '-999999px';
textarea.style.top = '-999999px';
document.body.appendChild(textarea);
textarea.focus();
textarea.select();
const success = document.execCommand('copy');
document.body.removeChild(textarea);
if (success) {
resolve(true);
} else {
resolve(false);
}
} catch (e) {
console.error("Clipboard operations failed:", e);
resolve(false);
}
});
} else {
// No clipboard API, try execCommand directly
try {
const textarea = document.createElement('textarea');
textarea.value = text;
// Position off-screen but available
textarea.style.position = 'fixed';
textarea.style.left = '-999999px';
textarea.style.top = '-999999px';
document.body.appendChild(textarea);
textarea.focus();
textarea.select();
const success = document.execCommand('copy');
document.body.removeChild(textarea);
if (success) {
resolve(true);
} else {
resolve(false);
}
} catch (e) {
console.error("Clipboard operations failed:", e);
resolve(false);
}
}
});
}
// Enhanced API request function with retry logic, rate limiting, and maxlag handling
async function makeApiRequest(url, retryCount = 0) {
try {
await new Promise(resolve => setTimeout(resolve, API_DELAY));
const response = await fetch(url);
// Handle rate limiting (HTTP 429) or server errors (5xx)
if (response.status === 429 || response.status >= 500) {
if (retryCount < MAX_RETRIES) {
const waitTime = Math.pow(2, retryCount) * 1000; // Exponential backoff
statusText.innerHTML += `<br>Rate limited or server error, waiting ${waitTime/1000}s before retry ${retryCount + 1}/${MAX_RETRIES}...`;
await new Promise(resolve => setTimeout(resolve, waitTime));
return makeApiRequest(url, retryCount + 1);
} else {
throw new Error(`Request failed after ${MAX_RETRIES} retries: ${response.status}`);
}
}
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
const data = await response.json();
// Handle maxlag errors - these don't count as retries since they're not real failures
if (data.error && data.error.code === 'maxlag') {
const lagTime = data.error.lag || 5; // Default to 5 seconds if lag not specified
const waitTime = (lagTime + 2) * 1000; // Add 2 second buffer
statusText.innerHTML += `<br>Database lagged (${lagTime}s), waiting ${waitTime/1000}s before retry...`;
await new Promise(resolve => setTimeout(resolve, waitTime));
return makeApiRequest(url, retryCount); // Don't increment retry count for maxlag
}
// Handle other API errors
if (data.error) {
throw new Error(`API Error: ${data.error.code} - ${data.error.info}`);
}
return data;
} catch (error) {
if (retryCount < MAX_RETRIES) {
statusText.innerHTML += `<br>Request failed, retrying ${retryCount + 1}/${MAX_RETRIES}...`;
await new Promise(resolve => setTimeout(resolve, 1000));
return makeApiRequest(url, retryCount + 1);
} else {
throw error;
}
}
}
// Function to get all subcategories of a category
async function getSubcategories(categoryTitle, continueToken = null) {
try {
// Base API URL for subcategories (only get items with namespace 14, which is Category)
// Add maxlag parameter to be respectful of server load
let apiUrl = `https://en.wikipedia.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:${encodeURIComponent(categoryTitle)}&cmnamespace=14&cmlimit=max&maxlag=5&format=json&origin=*`;
// Add continue token if provided
if (continueToken) {
apiUrl += `&cmcontinue=${continueToken}`;
}
statusText.textContent = `Fetching subcategories for: ${categoryTitle}...`;
const data = await makeApiRequest(apiUrl);
if (!data.query || !data.query.categorymembers) {
console.error("Unexpected API response:", data);
return { subcategories: [], continueToken: null };
}
// Extract subcategories and continue token, prefix with "Category:"
const subcategories = data.query.categorymembers.map(member => member.title); // Keep full "Category:" prefix
const nextContinueToken = data.continue ? data.continue.cmcontinue : null;
return { subcategories, continueToken: nextContinueToken };
} catch (error) {
console.error("API request error:", error);
statusText.innerHTML += `<br>Error fetching subcategories: ${error.message}`;
return { subcategories: [], continueToken: null };
}
}
// Function to get all non-category members of a category
async function getNonCategoryMembers(categoryTitle, continueToken = null) {
try {
// Base API URL for non-category members (exclude namespace 14, which is Category)
// Add maxlag parameter to be respectful of server load
let apiUrl = `https://en.wikipedia.org/w/api.php?action=query&list=categorymembers&cmtitle=Category:${encodeURIComponent(categoryTitle)}&cmnamespace=0|1|2|3|4|5|6|7|8|9|10|11|12|13|15&cmlimit=max&maxlag=5&format=json&origin=*`;
// Add continue token if provided
if (continueToken) {
apiUrl += `&cmcontinue=${continueToken}`;
}
statusText.textContent = `Fetching items for: ${categoryTitle}...`;
const data = await makeApiRequest(apiUrl);
if (!data.query || !data.query.categorymembers) {
console.error("Unexpected API response:", data);
return { members: [], continueToken: null };
}
// Extract members
const members = data.query.categorymembers.map(member => member.title);
const nextContinueToken = data.continue ? data.continue.cmcontinue : null;
return { members, continueToken: nextContinueToken };
} catch (error) {
console.error("API request error:", error);
statusText.innerHTML += `<br>Error fetching items: ${error.message}`;
return { members: [], continueToken: null };
}
}
// Function to get all members of a category, handling pagination
async function getAllCategoryMembers(categoryTitle) {
let allMembers = [];
let continueToken = null;
let pagesProcessed = 0;
do {
const { members, continueToken: nextToken } = await getNonCategoryMembers(categoryTitle, continueToken);
allMembers = allMembers.concat(members);
continueToken = nextToken;
pagesProcessed++;
statusText.innerHTML = `Retrieved ${allMembers.length} items from "${categoryTitle}" (page ${pagesProcessed})...`;
} while (continueToken);
return allMembers;
}
// Function to get all subcategories of a category, handling pagination
async function getAllSubcategories(categoryTitle) {
let allSubcategories = [];
let continueToken = null;
let pagesProcessed = 0;
do {
const { subcategories, continueToken: nextToken } = await getSubcategories(categoryTitle, continueToken);
allSubcategories = allSubcategories.concat(subcategories);
continueToken = nextToken;
pagesProcessed++;
} while (continueToken);
return allSubcategories;
}
// Function to recursively get all subcategories with circular reference detection
async function getAllSubcategoriesRecursive(categoryTitle) {
const visited = new Set();
const allSubcategories = [];
const queue = [`Category:${categoryTitle}`]; // Start with prefixed category
while (queue.length > 0) {
const currentCategory = queue.shift();
// Skip if already visited (circular reference detection)
if (visited.has(currentCategory) || globalVisited.has(currentCategory)) {
continue;
}
visited.add(currentCategory);
globalVisited.add(currentCategory);
statusText.innerHTML = `Exploring subcategories (found ${allSubcategories.length} categories, queue: ${queue.length})...`;
// Get direct subcategories (remove "Category:" prefix for API call)
const categoryNameForApi = currentCategory.replace('Category:', '');
const directSubcategories = await getAllSubcategories(categoryNameForApi);
// Add new subcategories to results and queue
for (const subcategory of directSubcategories) {
if (!visited.has(subcategory) && !globalVisited.has(subcategory)) {
allSubcategories.push(subcategory);
queue.push(subcategory);
}
}
}
return allSubcategories;
}
// Function to recursively get all items from a category and all its subcategories
async function getAllItemsRecursive(categoryTitle) {
const visited = new Set();
const allItems = [];
const queue = [categoryTitle]; // Start without prefix for consistency
let totalCategories = 0;
while (queue.length > 0) {
const currentCategory = queue.shift();
const categoryKey = `Category:${currentCategory}`;
// Skip if already visited (circular reference detection)
if (visited.has(categoryKey) || globalVisited.has(categoryKey)) {
continue;
}
visited.add(categoryKey);
globalVisited.add(categoryKey);
totalCategories++;
statusText.innerHTML = `Getting items from "${currentCategory}" (processed ${totalCategories} categories, found ${allItems.length} items, queue: ${queue.length})...`;
// Get items from current category
const currentItems = await getAllCategoryMembers(currentCategory);
allItems.push(...currentItems);
// Get direct subcategories and add to queue
const directSubcategories = await getAllSubcategories(currentCategory);
for (const subcategory of directSubcategories) {
if (!visited.has(subcategory) && !globalVisited.has(subcategory)) {
// Remove "Category:" prefix for queue consistency
const subcategoryName = subcategory.replace('Category:', '');
queue.push(subcategoryName);
}
}
}
return { items: allItems, totalCategories };
}
// Function to get both items and subcategories from a category (non-recursive)
async function getBothItemsAndSubcategories(categoryTitle) {
statusText.innerHTML = 'Gathering items and subcategories from this category...';
const items = await getAllCategoryMembers(categoryTitle);
const subcategories = await getAllSubcategories(categoryTitle);
return { items, subcategories };
}
// Function to recursively get both items and subcategories from a category and all its subcategories
async function getBothItemsAndSubcategoriesRecursive(categoryTitle) {
const visited = new Set();
const allItems = [];
const allSubcategories = [];
const queue = [categoryTitle]; // Start without prefix for consistency
let totalCategories = 0;
while (queue.length > 0) {
const currentCategory = queue.shift();
const categoryKey = `Category:${currentCategory}`;
// Skip if already visited (circular reference detection)
if (visited.has(categoryKey) || globalVisited.has(categoryKey)) {
continue;
}
visited.add(categoryKey);
globalVisited.add(categoryKey);
totalCategories++;
statusText.innerHTML = `Getting items and subcategories from "${currentCategory}" (processed ${totalCategories} categories, found ${allItems.length} items, ${allSubcategories.length} subcategories, queue: ${queue.length})...`;
// Get items from current category
const currentItems = await getAllCategoryMembers(currentCategory);
allItems.push(...currentItems);
// Get direct subcategories
const directSubcategories = await getAllSubcategories(currentCategory);
// Add subcategories to results and queue
for (const subcategory of directSubcategories) {
if (!visited.has(subcategory) && !globalVisited.has(subcategory)) {
allSubcategories.push(subcategory);
// Remove "Category:" prefix for queue consistency
const subcategoryName = subcategory.replace('Category:', '');
queue.push(subcategoryName);
}
}
}
return { items: allItems, subcategories: allSubcategories, totalCategories };
}
// Handle "Copy Items" button click
copyItemsBtn.addEventListener('click', async () => {
statusText.innerHTML = 'Gathering items from this category via API...';
try {
const items = await getAllCategoryMembers(categoryName);
if (items.length === 0) {
statusText.innerHTML = 'No items found in this category.';
return;
}
const includeUrls = urlCheckbox.checked;
const formattedText = formatItems(items, includeUrls);
const copySuccess = await copyToClipboardOrDownload(formattedText, categoryName);
if (copySuccess) {
statusText.innerHTML = `Successfully copied ${items.length} items to clipboard.`;
}
} catch (error) {
statusText.innerHTML = `Error: ${error.message}`;
console.error('Error:', error);
}
});
// Handle "Copy All Items" button click
copyAllItemsBtn.addEventListener('click', async () => {
statusText.innerHTML = 'Gathering items from this category and all subcategories recursively via API (this may take a while)...';
try {
// Clear global visited set for this operation
globalVisited.clear();
// Get all items recursively
const { items: allItems, totalCategories } = await getAllItemsRecursive(categoryName);
// Deduplicate items
const uniqueItems = [...new Set(allItems)];
if (uniqueItems.length === 0) {
statusText.innerHTML = 'No items found in this category or its subcategories.';
return;
}
const includeUrls = urlCheckbox.checked;
const formattedText = formatItems(uniqueItems, includeUrls);
const copySuccess = await copyToClipboardOrDownload(formattedText, categoryName + '_all_recursive');
if (copySuccess) {
statusText.innerHTML = `Successfully copied ${uniqueItems.length} unique items to clipboard from ${totalCategories} categories.`;
}
} catch (error) {
statusText.innerHTML = `Error: ${error.message}`;
console.error('Error:', error);
}
});
// Handle "Copy Subcats from this Category" button click
copyDirectSubcatsBtn.addEventListener('click', async () => {
statusText.innerHTML = 'Gathering direct subcategories from this category via API...';
try {
const subcategories = await getAllSubcategories(categoryName);
if (subcategories.length === 0) {
statusText.innerHTML = 'No direct subcategories found in this category.';
return;
}
const includeUrls = urlCheckbox.checked;
const formattedText = formatItems(subcategories, includeUrls);
const copySuccess = await copyToClipboardOrDownload(formattedText, categoryName + '_direct_subcats');
if (copySuccess) {
statusText.innerHTML = `Successfully copied ${subcategories.length} direct subcategories to clipboard.`;
}
} catch (error) {
statusText.innerHTML = `Error: ${error.message}`;
console.error('Error:', error);
}
});
// Handle "Copy Subcategories" button click
copySubcatsBtn.addEventListener('click', async () => {
statusText.innerHTML = 'Gathering all subcategories recursively via API (this may take a while)...';
try {
// Clear global visited set for this operation
globalVisited.clear();
const allSubcategories = await getAllSubcategoriesRecursive(categoryName);
// Deduplicate subcategories
const uniqueSubcategories = [...new Set(allSubcategories)];
if (uniqueSubcategories.length === 0) {
statusText.innerHTML = 'No subcategories found.';
return;
}
const includeUrls = urlCheckbox.checked;
const formattedText = formatItems(uniqueSubcategories, includeUrls);
const copySuccess = await copyToClipboardOrDownload(formattedText, categoryName + '_subcategories');
if (copySuccess) {
statusText.innerHTML = `Successfully copied ${uniqueSubcategories.length} unique subcategories to clipboard.`;
}
} catch (error) {
statusText.innerHTML = `Error: ${error.message}`;
console.error('Error:', error);
}
});
// Handle "Copy Both" button click
copyBothBtn.addEventListener('click', async () => {
statusText.innerHTML = 'Gathering both items and subcategories from this category via API...';
try {
const { items, subcategories } = await getBothItemsAndSubcategories(categoryName);
if (items.length === 0 && subcategories.length === 0) {
statusText.innerHTML = 'No items or subcategories found in this category.';
return;
}
// Combine items and subcategories
const combinedResults = [...items, ...subcategories];
const includeUrls = urlCheckbox.checked;
const formattedText = formatItems(combinedResults, includeUrls);
const copySuccess = await copyToClipboardOrDownload(formattedText, categoryName + '_both');
if (copySuccess) {
statusText.innerHTML = `Successfully copied ${items.length} items and ${subcategories.length} subcategories (${combinedResults.length} total) to clipboard.`;
}
} catch (error) {
statusText.innerHTML = `Error: ${error.message}`;
console.error('Error:', error);
}
});
// Handle "Copy Both Recursively" button click
copyBothRecursiveBtn.addEventListener('click', async () => {
statusText.innerHTML = 'Gathering both items and subcategories recursively via API (this may take a while)...';
try {
// Clear global visited set for this operation
globalVisited.clear();
const { items: allItems, subcategories: allSubcategories, totalCategories } = await getBothItemsAndSubcategoriesRecursive(categoryName);
// Deduplicate items and subcategories
const uniqueItems = [...new Set(allItems)];
const uniqueSubcategories = [...new Set(allSubcategories)];
if (uniqueItems.length === 0 && uniqueSubcategories.length === 0) {
statusText.innerHTML = 'No items or subcategories found in this category or its subcategories.';
return;
}
// Combine items and subcategories
const combinedResults = [...uniqueItems, ...uniqueSubcategories];
const includeUrls = urlCheckbox.checked;
const formattedText = formatItems(combinedResults, includeUrls);
const copySuccess = await copyToClipboardOrDownload(formattedText, categoryName + '_both_recursive');
if (copySuccess) {
statusText.innerHTML = `Successfully copied ${uniqueItems.length} unique items and ${uniqueSubcategories.length} unique subcategories (${combinedResults.length} total) to clipboard from ${totalCategories} categories.`;
}
} catch (error) {
statusText.innerHTML = `Error: ${error.message}`;
console.error('Error:', error);
}
});
console.log('Wikipedia Category Copier script has been loaded successfully!');
} else {
console.log('Wikipedia Category Copier: Not a category page, script inactive.');
}
