import { randomBytes } from 'crypto';
import { addUpload } from "./addUpload.js";
import mime from 'mime-types';
/**
* Check if a URL is a foreign (external) image URL
* Returns false for:
* - Relative URLs (./uploads/, uploads/, ../etc)
* - Data URLs (data:image/...)
* - Same-origin URLs (handled by checking protocol)
*/
function isForeignUrl(url) {
// Trim the URL
const trimmed = url.trim();
// Skip empty URLs
if (!trimmed)
return false;
// Skip data URLs
if (trimmed.startsWith('data:'))
return false;
// Skip relative URLs (uploads, ./, ../, etc)
if (trimmed.startsWith('./') || trimmed.startsWith('../') || trimmed.startsWith('/'))
return false;
if (trimmed.startsWith('uploads/'))
return false;
// Check for absolute URLs with http/https protocol
try {
const parsed = new URL(trimmed);
return parsed.protocol === 'http:' || parsed.protocol === 'https:';
}
catch {
// Not a valid URL, likely relative
return false;
}
}
/**
* Extract the filename from a URL
* Falls back to generating a random filename if extraction fails
*/
function extractFilenameFromUrl(url) {
try {
const parsed = new URL(url);
const pathname = parsed.pathname;
// Get the last segment of the path
const segments = pathname.split('/').filter(Boolean);
if (segments.length > 0) {
const lastSegment = segments[segments.length - 1];
// Decode URL-encoded characters
const decoded = decodeURIComponent(lastSegment);
// Remove query strings if present (shouldn't be, but just in case)
const cleaned = decoded.split('?')[0];
if (cleaned && cleaned.length > 0 && cleaned.length < 255) {
// Ensure it has an extension, otherwise add .jpg as default
if (!cleaned.includes('.')) {
return `${cleaned}.jpg`;
}
return cleaned;
}
}
}
catch {
// Fall through to random name
}
// Generate random filename with .jpg extension
return `image-${randomBytes(4).toString('hex')}.jpg`;
}
/**
* Extract foreign images from HTML content
* Matches
patterns
*/
function extractHtmlImages(content) {
const images = [];
// Match
tags with src attribute
// Handles: src="url", src='url', src=url (unquoted)
// Also captures alt attribute if present
const imgRegex = /
]*?src\s*=\s*(?:["']([^"']+)["']|([^\s>]+))[^>]*?>/gi;
let match;
while ((match = imgRegex.exec(content)) !== null) {
const url = match[1] || match[2];
if (url && isForeignUrl(url)) {
// Try to extract alt text
const altMatch = match[0].match(/alt\s*=\s*(?:["']([^"']*)["']|([^\s>]+))/i);
const alt = altMatch ? altMatch[1] || altMatch[2] : undefined;
images.push({
url,
match: match[0],
type: 'html',
alt,
});
}
}
return images;
}
/**
* Extract foreign images from Markdown content
* Matches  patterns
*/
function extractMarkdownImages(content) {
const images = [];
// Match markdown image syntax:  or 
// Also handles empty alt: 
const mdImageRegex = /!\[([^\]]*)\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
let match;
while ((match = mdImageRegex.exec(content)) !== null) {
const alt = match[1];
const url = match[2];
if (url && isForeignUrl(url)) {
images.push({
url,
match: match[0],
type: 'markdown',
alt: alt || undefined,
});
}
}
return images;
}
/**
* Extract all foreign images from content (both HTML and Markdown)
*/
function extractForeignImages(content) {
const htmlImages = extractHtmlImages(content);
const markdownImages = extractMarkdownImages(content);
// Deduplicate by URL
const seen = new Set();
const result = [];
for (const img of [...htmlImages, ...markdownImages]) {
if (!seen.has(img.url)) {
seen.add(img.url);
result.push(img);
}
}
return result;
}
/**
* Get file extension from content type using mime-types module
*/
function getExtensionFromContentType(contentType) {
if (!contentType)
return null;
const type = contentType.split(';')[0].trim().toLowerCase();
const ext = mime.extension(type);
return ext ? `.${ext}` : null;
}
/**
* Download a single image from a URL
*/
async function downloadImage(image, options = {}) {
const { timeout = 10000 } = options;
try {
const controller = new AbortController();
const timeoutId = setTimeout(() => controller.abort(), timeout);
const response = await fetch(image.url, {
signal: controller.signal,
headers: {
// Some servers require a user agent
'User-Agent': 'Mozilla/5.0 (compatible; Skywriter/1.0; +https://github.com/reggi/skywriter) Image Downloader',
},
});
clearTimeout(timeoutId);
if (!response.ok) {
console.warn(`Failed to download image ${image.url}: HTTP ${response.status}`);
return null;
}
const contentType = response.headers.get('content-type');
// Verify it's an image
if (contentType && !contentType.startsWith('image/')) {
console.warn(`URL ${image.url} did not return an image: ${contentType}`);
return null;
}
const arrayBuffer = await response.arrayBuffer();
const data = Buffer.from(arrayBuffer);
// Determine filename
let filename = extractFilenameFromUrl(image.url);
// If the filename doesn't have a proper extension, try to get one from content type
const hasValidExtension = /\.(jpe?g|png|gif|webp|svg|bmp|tiff?|ico|avif|heic|heif)$/i.test(filename);
if (!hasValidExtension) {
const ext = getExtensionFromContentType(contentType);
if (ext) {
// Replace or add extension
const base = filename.replace(/\.[^.]+$/, '');
filename = base + ext;
}
}
return {
original: image,
data,
contentType,
filename,
};
}
catch (error) {
if (error instanceof Error && error.name === 'AbortError') {
console.warn(`Timeout downloading image ${image.url}`);
}
else {
console.warn(`Error downloading image ${image.url}:`, error);
}
return null;
}
}
/**
* Download multiple images in parallel with concurrency limit
*/
async function downloadImages(images, options = {}) {
const { concurrency = 5 } = options;
const results = [];
const queue = [...images];
async function worker() {
while (queue.length > 0) {
const image = queue.shift();
if (image) {
const result = await downloadImage(image, options);
if (result) {
results.push(result);
}
}
}
}
// Start concurrent workers
const workers = Array.from({ length: Math.min(concurrency, images.length) }, () => worker());
await Promise.all(workers);
return results;
}
/**
* Replace foreign image URLs in content with local upload URLs
*/
function replaceForeignUrls(content, savedUploads) {
let result = content;
for (const saved of savedUploads) {
const { original, localUrl } = saved;
if (original.type === 'html') {
// Replace the src attribute value in the img tag
// We need to carefully replace just the URL, not the entire tag
// Replace the URL within the matched img tag
const newTag = original.match.replace(original.url, localUrl);
result = result.replace(original.match, newTag);
}
else {
// Markdown: replace the full  with 
const alt = original.alt || '';
const newMarkdown = ``;
result = result.replace(original.match, newMarkdown);
}
}
return result;
}
/**
* Process content to download foreign images and replace URLs
* Returns the modified content with foreign images replaced by local uploads
*
* This function:
* 1. Extracts foreign image URLs from HTML and Markdown content
* 2. Downloads the images in parallel with concurrency limit
* 3. Saves them as uploads using addUpload
* 4. Replaces the original URLs with local upload URLs
*
* @param client Database client
* @param query Document query (supports path string, id number, OptimisticDocument, Route, etc.)
* @param uploadsPath Path to the uploads directory
* @param content The content to process for foreign images
* @returns Object with modified content and list of saved uploads
*/
export const addUploadsFromContent = async (client, query, uploadsPath, content) => {
// Extract foreign images from content
const foreignImages = extractForeignImages(content);
if (foreignImages.length === 0) {
return { content, uploads: [] };
}
console.log(`Found ${foreignImages.length} foreign image(s) to process`);
// Download images
const downloadedImages = await downloadImages(foreignImages);
if (downloadedImages.length === 0) {
console.log('No images were successfully downloaded');
return { content, uploads: [] };
}
console.log(`Downloaded ${downloadedImages.length} image(s)`);
// Save as uploads
const savedUploads = [];
for (const downloaded of downloadedImages) {
try {
// Use consolidated addUpload which handles:
// - Unique storage filename generation
// - Unique original_filename generation
// - Database record creation
// - File writing to disk
const upload = await addUpload(client, query, uploadsPath, {
data: downloaded.data,
filename: downloaded.filename,
});
// Create local URL (relative)
const localUrl = `./uploads/${encodeURIComponent(upload.original_filename)}`;
savedUploads.push({
original: downloaded.original,
localUrl,
originalFilename: upload.original_filename,
});
}
catch (error) {
console.error(`Failed to save upload for ${downloaded.original.url}:`, error);
}
}
if (savedUploads.length === 0) {
console.log('No images were saved as uploads');
return { content, uploads: [] };
}
console.log(`Saved ${savedUploads.length} image(s) as uploads`);
// Replace URLs in content
const modifiedContent = replaceForeignUrls(content, savedUploads);
return { content: modifiedContent, uploads: savedUploads };
};
//# sourceMappingURL=addUploadsFromContent.js.map