stephen/124_webcalibre2
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

not full extracting metadata

Browse source
This commit is contained in:
stephen 2026-02-01 21:37:02 +11:00
parent 9552846a0f
commit 71b567ad04
6 changed files with 109 additions and 3067 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2873
docs/notes_tmp.html
View file

File diff suppressed because it is too large Load diff

123
src/app/dashboard/actions.ts
View file

@ -16,12 +16,14 @@ import {
uploadToOneDrive
} from "@/services/onedrive";
import { getEnrichedMetadataFromCloud } from "@/data-access/file-nodes";
//import { getEnrichedMetadataFromCloud } from "@/data-access/file-nodes";
import { getFreshAccessToken } from "@/lib/auth-utils";
import { getOneDriveFileBuffer } from "@/services/onedrive"; // Import your working service
import { extractMetadata } from "@/lib/metadata-extractor";
/**
* 1. FETCH: Get all file nodes
* Now simply calls the DAL. Error handling is left to the caller (the UI).
@ -120,134 +122,35 @@ export async function updateFileNodeAction(id: string, formData: FormData) {
}
}
// export async function getMetadataPreviewAction(fileId: string) {
// const session = await auth();
// if (!session?.user?.id) throw new Error("Unauthorized");
// try {
// console.log(`🔍 Starting enhancement for file: ${fileId}`);
// // This calls the DAL -> which calls the Service -> which calls OneDrive
// const data = await getEnrichedMetadataFromCloud(fileId);
// // This log will show you exactly what we found in your terminal!
// console.log("✅ Extracted Metadata Result:", data);
// return { success: true, data };
// } catch (error: any) {
// console.error("❌ Enhancement Action Error:", error.message);
// return { success: false, error: error.message };
// }
// }
/**
* 5. ENHANCE (Magic Fill): Extracts deep metadata from the actual file binary
*/
// export async function getMetadataPreviewAction(fileId: string) {
// const session = await auth();
// if (!session?.user?.id) throw new Error("Unauthorized");
// console.log(`\n--- 🔍 Magic Fill Started for File ID: ${fileId} ---`);
// try {
// // 1. Get the record from DB to get the oneDriveId and Name
// const node = await getFileNodeById(fileId);
// if (!node || !node.oneDriveId) {
// throw new Error("File not found or not synced with OneDrive");
// }
// // 2. Get fresh token
// const token = await getFreshAccessToken(session.user.id);
// // 3. Fetch the actual binary content from Microsoft Graph
// console.log(`📡 Fetching binary from Microsoft Graph...`);
// let response;
// try {
// response = await fetch(
// `https://graph.microsoftonline.com/v1.0/me/drive/items/${node.oneDriveId}/content`,
// {
// headers: { Authorization: `Bearer ${token}` },
// cache: 'no-store' // Ensure we aren't hitting a stale server cache
// }
// );
// } catch (err: any) {
// console.error("❌ THE ACTUAL NETWORK ERROR:");
// console.error("Message:", err.message);
// console.error("Cause/Stack:", err.cause || err.stack); // This is the gold mine
// throw new Error(`Server-side fetch failed: ${err.message}`);
// }
// if (!response.ok) {
// throw new Error(`Failed to fetch file content: ${response.statusText}`);
// }
// const arrayBuffer = await response.arrayBuffer();
// const buffer = Buffer.from(arrayBuffer);
// console.log(`📦 Downloaded ${buffer.length} bytes.`);
// // 4. Run the Metadata Utility
// const extractedData = await extractMetadata(buffer, node.name);
// // --- 🏁 THE TERMINAL LOG YOU REQUESTED ---
// console.log("✅ RAW DATA EXTRACTED FROM FILE:");
// console.dir(extractedData, { depth: null, colors: true });
// console.log(`--- 🏁 Magic Fill Finished ---\n`);
// return { success: true, data: extractedData };
// } catch (error: any) {
// console.error("❌ Magic Fill Error:", error.message);
// return { success: false, error: error.message };
// }
// }
export async function getMetadataPreviewAction(fileId: string) {
const session = await auth();
if (!session?.user?.id) throw new Error("Unauthorized");
console.log(`\n--- 🔍 Magic Fill Started for File ID: ${fileId} ---`);
try {
const node = await getFileNodeById(fileId);
if (!node || !node.oneDriveId) throw new Error("File not found");
if (!node || !node.oneDriveId) throw new Error("No OneDrive ID found");
console.log(`📡 Attempting fetch via Service for: ${node.name}`);
// 1. Get the token from our utility
const token = await getFreshAccessToken(session.user.id);
// LOG: Just check the length to be sure it's not empty
console.log(`🔑 Token retrieved (Length: ${token.length})`);
if (!token) throw new Error("Could not retrieve access token");
// Call your existing service
const buffer = await getOneDriveFileBuffer(node.oneDriveId, token);
console.log(`📡 Fetching binary for: ${node.name}...`);
console.log(`📦 Buffer received: ${buffer.length} bytes`);
// 2. Fetch the content from Microsoft Graph
const response = await fetch(
`https://graph.microsoft.com/v1.0/me/drive/items/${node.oneDriveId}/content`,
{
headers: {
'Authorization': `Bearer ${token}`,
'Accept': '*/*'
},
}
);
if (!response.ok) {
// If it fails here, we'll see the real reason from Microsoft
const errorText = await response.text();
console.error("❌ Microsoft Graph Error Response:", errorText);
throw new Error(`OneDrive Download Failed: ${response.status}`);
}
const arrayBuffer = await response.arrayBuffer();
const buffer = Buffer.from(arrayBuffer);
console.log(`📦 Success! Downloaded ${buffer.length} bytes.`);
// 3. Extract Metadata
const extractedData = await extractMetadata(buffer, node.name);
console.log("✅ RAW DATA EXTRACTED:");
console.dir(extractedData, { depth: null, colors: true });
console.log("✅ Extracted:", extractedData);
return { success: true, data: extractedData };
} catch (error: any) {
console.error("❌ Magic Fill Error:", error.message);
console.error("❌ Service Fetch Error:", error.message);
// If this still says ENOTFOUND, the code is fine, but the terminal is blocked.
return { success: false, error: error.message };
}
}

20
src/data-access/file-nodes.ts
View file

@ -109,23 +109,3 @@ export async function upsertFileNode(oneDriveId: string, data: any) {
/**
* Logic to coordinate getting a file from the cloud and extracting its data.
* This is the "Brain" function for your metadata enrichment.
*/
export async function getEnrichedMetadataFromCloud(fileId: string) {
// 1. Get the record from our DB so we know the filename (needed for extension logic)
const node = await prisma.fileNode.findUnique({
where: { id: fileId }
});
if (!node) throw new Error("File not found in database.");
// 2. Fetch the bytes using the service we just created
const buffer = await getOneDriveFileBuffer(fileId);
// 3. Extract internal metadata (Title, Author, or GPS coordinates)
const deepMetadata = await extractMetadata(buffer, node.name);
return deepMetadata;
}

74
src/lib/metadata-extractor.ts
View file

@ -11,19 +11,44 @@ export interface ExtractedMetadata {
pageCount?: number;
latitude?: number;
longitude?: number;
deviceModel?: string | null;
exposureTime?: string | null;
fNumber?: number | null;
iso?: number | null;
type: string;
}
/**
* Helper: Converts [Degrees, Minutes, Seconds] to Decimal
*/
function parseGps(values: number[] | undefined, ref: string | undefined): number | null {
if (!values || values.length < 3) return null;
const decimal = values[0] + values[1] / 60 + values[2] / 3600;
const multiplier = (ref === 'S' || ref === 'W') ? -1 : 1;
return Number((decimal * multiplier).toFixed(6));
}
/**
* Helper: Converts Rational fractions (1/4000) or Objects to readable values
*/
function parseRational(value: any, asString = false): string | number | null {
if (!value) return null;
if (typeof value === 'number') return value;
if (value.numerator && value.denominator) {
if (asString) return `${value.numerator}/${value.denominator}`;
return value.numerator / value.denominator;
}
return null;
}
export async function extractMetadata(buffer: Buffer, filename: string): Promise<ExtractedMetadata> {
const extension = filename.split('.').pop()?.toLowerCase();
try {
// --- 1. PDF EXTRACTION ---
if (extension === 'pdf') {
// Use any to bypass the missing 'default' property error in ESM
const parsePdf = (pdf as any).default || pdf;
const data = await parsePdf(buffer);
return {
type: 'PDF',
title: data.info?.Title || filename,
@ -33,44 +58,26 @@ export async function extractMetadata(buffer: Buffer, filename: string): Promise
};
}
// --- 2. EPUB EXTRACTION ---
if (extension === 'epub') {
// Logic for EPub usually requires file path or custom stream handler
// Keeping placeholder for your existing EPub logic
return { type: 'EPUB', title: filename };
}
// --- 3. IMAGE EXTRACTION (Enhanced with GPS) ---
// --- 2. IMAGE EXTRACTION (Enhanced) ---
if (['jpg', 'jpeg', 'png', 'webp'].includes(extension || '')) {
const image = sharp(buffer);
const metadata = await image.metadata();
let gps: { latitude?: number; longitude?: number } = {};
let exifData: Partial<ExtractedMetadata> = {};
if (metadata.exif) {
try {
// Cast to any to bypass strict Exif type checking for nested GPS properties
const exif = exifReader(metadata.exif) as any;
// Debugging log to see the raw structure in your terminal
console.log("📸 FULL RAW EXIF DATA:", JSON.stringify(exif, null, 2));
if (exif.gps && exif.gps.GPSLatitude && exif.gps.GPSLongitude) {
// EXIF stores GPS as [Degrees, Minutes, Seconds]
// We convert to Decimal Degrees for Google Maps
const lat = exif.gps.GPSLatitude;
const lon = exif.gps.GPSLongitude;
let latitude = lat[0] + lat[1] / 60 + lat[2] / 3600;
let longitude = lon[0] + lon[1] / 60 + lon[2] / 3600;
// Adjust for South or West hemisphere
if (exif.gps.GPSLatitudeRef === 'S') latitude *= -1;
if (exif.gps.GPSLongitudeRef === 'W') longitude *= -1;
gps.latitude = latitude;
gps.longitude = longitude;
}
console.log(" exif ");
console.log(exif);
exifData = {
deviceModel: exif.image?.Model || null,
exposureTime: parseRational(exif.photo?.ExposureTime, true) as string,
fNumber: parseRational(exif.photo?.FNumber) as number,
iso: exif.photo?.ISOSpeedRatings || null,
latitude: parseGps(exif.gps?.GPSLatitude, exif.gps?.GPSLatitudeRef) || undefined,
longitude: parseGps(exif.gps?.GPSLongitude, exif.gps?.GPSLongitudeRef) || undefined,
};
} catch (exifError) {
console.warn("Could not parse EXIF data for:", filename, exifError);
}
@ -80,10 +87,11 @@ export async function extractMetadata(buffer: Buffer, filename: string): Promise
type: `IMAGE (${metadata.format?.toUpperCase()})`,
dimensions: metadata.width && metadata.height ? `${metadata.width}x${metadata.height}` : undefined,
title: filename,
...gps
...exifData
};
}
// Default Fallback
return { type: 'FILE', title: filename };
} catch (error) {
console.error(`Extraction failed for ${filename}:`, error);

23
src/services/metadata-service.ts Normal file
View file

@ -0,0 +1,23 @@
// src/services/metadata-service.ts
import { getOneDriveFileBuffer } from "./onedrive";
// 1. Change the import name
import { extractMetadata } from "@/lib/metadata-extractor";
import { getFileNodeById, updateFileNode } from "@/data-access/file-nodes";
export async function enrichFileMetadata(fileId: string, token: string) {
const node = await getFileNodeById(fileId);
if (!node || !node.oneDriveId) throw new Error("Node not found");
const buffer = await getOneDriveFileBuffer(node.oneDriveId, token);
// 2. Use the standard extractor (it handles PDF, Image, etc. automatically)
const deepMetadata = await extractMetadata(buffer, node.name);
const updatedMetadata = {
...(node.metadata as object),
...deepMetadata, // This now contains type, dimensions, gps, deviceModel, etc.
magicFilled: true
};
return await updateFileNode(fileId, { metadata: updatedMetadata });
}

57
src/services/onedrive.ts
View file

@ -171,35 +171,7 @@ export async function uploadToFolderId(userId: string, file: File, folderId: str
if (!uploadRes.ok) throw new Error("Upload failed");
return await uploadRes.json();
}
/**
* Fetches the raw binary content (the actual file bytes) from OneDrive.
*/
export async function getOneDriveFileBuffer(fileId: string): Promise<Buffer> {
// Use your existing helper that manages the Microsoft Graph access token
const token = await getAccessToken();
const response = await fetch(
`https://graph.microsoft.com/v1.0/me/drive/items/${fileId}/content`,
{
method: 'GET',
headers: {
'Authorization': `Bearer ${token}`,
},
// Ensure we get fresh data and don't cache large file buffers
cache: 'no-store',
}
);
if (!response.ok) {
const errorText = await response.text();
console.error("OneDrive Download Error:", errorText);
throw new Error(`Failed to download file content: ${response.statusText}`);
}
// Convert the browser-style response into a Node.js Buffer
const arrayBuffer = await response.arrayBuffer();
return Buffer.from(arrayBuffer);
}
/**
@ -221,3 +193,32 @@ async function getAccessToken(): Promise<string> {
return token;
}
/**
* Fetches raw file content from OneDrive.
* Parameterized token allows this to be used in different contexts (User actions, Webhooks, etc.)
*/
export async function getOneDriveFileBuffer(oneDriveId: string, token: string): Promise<Buffer> {
const response = await fetch(
`https://graph.microsoft.com/v1.0/me/drive/items/${oneDriveId}/content`,
{
method: 'GET', // Explicit is better
headers: {
'Authorization': `Bearer ${token}`,
'Accept': '*/*'
},
// CRITICAL: Next.js tends to cache fetch calls.
// We do NOT want to cache large binary buffers in memory/disk.
cache: 'no-store',
}
);
if (!response.ok) {
const errorBody = await response.text().catch(() => "No error body");
console.error(`OneDrive Download Error (${response.status}):`, errorBody);
throw new Error(`OneDrive download failed: ${response.statusText}`);
}
const arrayBuffer = await response.arrayBuffer();
return Buffer.from(arrayBuffer);
}