pdf and image metadata extractor working
This commit is contained in:
parent
71b567ad04
commit
552662cf16
11 changed files with 481 additions and 189 deletions
|
|
@ -42,6 +42,9 @@
|
|||
- [1. Where to put the fetch call?](#1-where-to-put-the-fetch-call)
|
||||
- [2. Where to put `getMetadataFromJson`?](#2-where-to-put-getmetadatafromjson)
|
||||
- [🏗️ How it looks all together](#️-how-it-looks-all-together)
|
||||
- [image metadata](#image-metadata)
|
||||
- [pdf metadata](#pdf-metadata)
|
||||
- [useful](#useful)
|
||||
|
||||
# 1. Reference
|
||||
|
||||
|
|
@ -2495,3 +2498,34 @@ When you put these in their proper places, your **API Route** or **Server Action
|
|||
4. **Save to DB** (Prisma / DAL)
|
||||
|
||||
**Would you like me to help you refine the internal logic of the `extractExifMetadata` function to ensure it handles the specific "Rational" numbers (fractions) that EXIF often uses for exposure and GPS?**
|
||||
|
||||
|
||||
$\pi r^2$
|
||||
|
||||
$$H_2O$$
|
||||
|
||||
$$\sum$$
|
||||
|
||||
$$2^{10}$$
|
||||
|
||||
$$\sum_{i=1}^{n} i = \frac{n(n+1)}{2}$$
|
||||
|
||||
$$\int_{a}^{b} x^2 \, dx$$
|
||||
|
||||
$$\iiint$$
|
||||
|
||||
$$\oint$$
|
||||
|
||||
$\underline{\overline{ \text{--- } \bullet \text{ --- } \bullet \text{ --- } }}$
|
||||
|
||||
$\textifsym{mm<DDD>mm}$
|
||||
|
||||
$\FallingEdge$
|
||||
|
||||
# image metadata
|
||||
{"type": "JPEG", "mimeType": "image/jpeg", "dimensions": "4032x3024", "details.bigEndian": "true", "details.image.make": "Apple", "details.image.model": "iPhone 14", "details.photo.flash": "16", "details.image.gPSTag": "2586", "details.image.exifTag": "228", "details.photo.fNumber": "1.5", "details.image.dateTime": "2025-10-22T12:08:16.000Z", "details.image.software": "18.6.2", "details.photo.lensMake": "Apple", "details.photo.lensModel": "iPhone 14 back dual wide camera 5.7mm f/1.5", "details.photo.makerNote": "[Binary Data: 1710 bytes]", "details.photo.sceneType": "[Binary Data: 1 bytes]", "details.gPSInfo.gPSSpeed": "0.04381048133718738", "details.photo.colorSpace": "65535", "details.photo.offsetTime": "+02:00", "details.image.orientation": "6", "details.image.xResolution": "72", "details.image.yResolution": "72", "details.photo.exifVersion": "[Binary Data: 4 bytes]", "details.photo.focalLength": "5.7", "details.photo.subjectArea": "2209,1013,261,260", "details.image.hostComputer": "iPhone 14", "details.photo.exposureMode": "0", "details.photo.exposureTime": "0.001876172607879925", "details.photo.meteringMode": "5", "details.photo.whiteBalance": "0", "details.gPSInfo.gPSAltitude": "664.5505481120584", "details.gPSInfo.gPSLatitude": "40,25,5.48", "details.gPSInfo.gPSSpeedRef": "K", "details.photo.apertureValue": "1.1699250021066825", "details.photo.sensingMethod": "2", "details.gPSInfo.gPSDateStamp": "2025:10:22", "details.gPSInfo.gPSLongitude": "3,41,7.67", "details.gPSInfo.gPSTimeStamp": "10,8,15", "details.image.resolutionUnit": "2", "details.photo.compositeImage": "2", "details.photo.brightnessValue": "7.091505376344086", "details.photo.exposureProgram": "2", "details.photo.flashpixVersion": "[Binary Data: 4 bytes]", "details.photo.iSOSpeedRatings": "50", "details.photo.pixelXDimension": "4032", "details.photo.pixelYDimension": "3024", "details.thumbnail.compression": "6", "details.thumbnail.xResolution": "72", "details.thumbnail.yResolution": "72", "details.gPSInfo.gPSAltitudeRef": "0", "details.gPSInfo.gPSDestBearing": "345.1883852691218", "details.gPSInfo.gPSLatitudeRef": "N", "details.image.yCbCrPositioning": "1", "details.photo.dateTimeOriginal": "2025-10-22T12:08:16.000Z", "details.photo.sceneCaptureType": "0", "details.gPSInfo.gPSImgDirection": "345.1883852691218", "details.gPSInfo.gPSLongitudeRef": "W", "details.photo.dateTimeDigitized": "2025-10-22T12:08:16.000Z", "details.photo.exposureBiasValue": "0", "details.photo.lensSpecification": "1.5399999618512084,5.699999809263318,1.5,2.4", "details.photo.shutterSpeedValue": "9.058893693156405", "details.photo.offsetTimeOriginal": "+02:00", "details.photo.subSecTimeOriginal": "132", "details.thumbnail.resolutionUnit": "2", "details.gPSInfo.gPSDestBearingRef": "T", "details.photo.offsetTimeDigitized": "+02:00", "details.photo.subSecTimeDigitized": "132", "details.gPSInfo.gPSImgDirectionRef": "T", "details.photo.focalLengthIn35mmFilm": "26", "details.gPSInfo.gPSHPositioningError": "14.15490024117518", "details.photo.componentsConfiguration": "[Binary Data: 4 bytes]", "details.thumbnail.jPEGInterchangeFormat": "2990", "details.thumbnail.jPEGInterchangeFormatLength": "11238"}
|
||||
|
||||
# pdf metadata
|
||||
{"type": "PDF", "mimeType": "application/pdf", "pageCount": "1", "textPreview": " Hello World! 3 1", "details.title": "Analysis of Electromagnetic Field Circulation", "details.author": "Stephen Lohning", "details.creator": "pdfLaTeX", "details.modDate": "D:20260205160832+11'00'", "details.subject": "Electrical Engineering", "details.keywords": "Maxwell, Electromagnetics, Integral Form, EE", "details.language": "null", "details.producer": "LaTeX", "details.creationDate": "D:20260205160832+11'00'", "details.isLinearized": "false", "details.isXFAPresent": "false", "details.trapped.name": "False", "details.pDFFormatVersion": "1.7", "details.encryptFilterName": "null", "details.isAcroFormPresent": "false", "details.isCollectionPresent": "false", "details.isSignaturesPresent": "false", "details.custom.pTEX.Fullbanner": "This is pdfTeX, Version 3.141592653-2.6-1.40.27 (TeX Live 2025/Homebrew) kpathsea version 6.4.1"}
|
||||
|
||||
## useful
|
||||
BIN
docs/notes.pdf
BIN
docs/notes.pdf
Binary file not shown.
20
package-lock.json
generated
20
package-lock.json
generated
|
|
@ -23,6 +23,7 @@
|
|||
"next": "16.1.1",
|
||||
"next-auth": "^5.0.0-beta.30",
|
||||
"pdf-parse": "^2.4.5",
|
||||
"pdf-parse-new": "^2.0.0",
|
||||
"pg": "^8.16.3",
|
||||
"react": "19.2.3",
|
||||
"react-dom": "19.2.3",
|
||||
|
|
@ -6053,6 +6054,12 @@
|
|||
}
|
||||
}
|
||||
},
|
||||
"node_modules/node-ensure": {
|
||||
"version": "0.0.0",
|
||||
"resolved": "https://registry.npmjs.org/node-ensure/-/node-ensure-0.0.0.tgz",
|
||||
"integrity": "sha512-DRI60hzo2oKN1ma0ckc6nQWlHU69RH6xN0sjQTjMpChPfTYvKZdcQFfdYK2RWbJcKyUizSIy/l8OTGxMAM1QDw==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/node-fetch-native": {
|
||||
"version": "1.6.7",
|
||||
"resolved": "https://registry.npmjs.org/node-fetch-native/-/node-fetch-native-1.6.7.tgz",
|
||||
|
|
@ -6550,6 +6557,19 @@
|
|||
"url": "https://github.com/sponsors/mehmet-kozan"
|
||||
}
|
||||
},
|
||||
"node_modules/pdf-parse-new": {
|
||||
"version": "2.0.0",
|
||||
"resolved": "https://registry.npmjs.org/pdf-parse-new/-/pdf-parse-new-2.0.0.tgz",
|
||||
"integrity": "sha512-9RG2m8lws0H/6kn9kngrooLNJM/FW8rtLRLgtxd69fuEV/uuP7L6ez3dKt2hGW/laO/eVxGiQYYwo+wysB4saA==",
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"debug": "^4.3.4",
|
||||
"node-ensure": "^0.0.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20.11.0"
|
||||
}
|
||||
},
|
||||
"node_modules/pdfjs-dist": {
|
||||
"version": "5.4.296",
|
||||
"resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-5.4.296.tgz",
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@
|
|||
"next": "16.1.1",
|
||||
"next-auth": "^5.0.0-beta.30",
|
||||
"pdf-parse": "^2.4.5",
|
||||
"pdf-parse-new": "^2.0.0",
|
||||
"pg": "^8.16.3",
|
||||
"react": "19.2.3",
|
||||
"react-dom": "19.2.3",
|
||||
|
|
|
|||
|
|
@ -1,12 +1,11 @@
|
|||
'use client';
|
||||
|
||||
// src/app/update/[id]/update-view.tsx
|
||||
|
||||
import { useState } from "react";
|
||||
import {
|
||||
Box, Button, Typography, Paper, Stack,
|
||||
TextField, MenuItem, IconButton, Grid, Divider,
|
||||
Checkbox, CircularProgress, Chip, Tooltip
|
||||
TextField, MenuItem, IconButton, Grid,
|
||||
Checkbox, CircularProgress, Tooltip
|
||||
} from "@mui/material";
|
||||
import SaveIcon from "@mui/icons-material/Save";
|
||||
import ArrowBackIcon from '@mui/icons-material/ArrowBack';
|
||||
|
|
@ -22,29 +21,37 @@ import { getMetadataPreviewAction } from "@/app/dashboard/actions";
|
|||
interface MetadataPair {
|
||||
key: string;
|
||||
value: string;
|
||||
selected: boolean; // Checkbox state
|
||||
isPending?: boolean; // Visual highlight for auto-extracted fields
|
||||
selected: boolean;
|
||||
isPending?: boolean;
|
||||
}
|
||||
|
||||
export default function UpdateView({
|
||||
fileNode,
|
||||
folders: availablefolders // Renaming 'folders' to 'availablefolders'
|
||||
}: {
|
||||
fileNode: any;
|
||||
folders: any[];
|
||||
}) {
|
||||
/** * Utility to turn nested objects into flat key-value pairs for the UI
|
||||
*/
|
||||
const flattenObject = (obj: any, prefix = ''): Record<string, string> => {
|
||||
let results: Record<string, string> = {};
|
||||
for (const key in obj) {
|
||||
const value = obj[key];
|
||||
const newKey = prefix ? `${prefix}.${key}` : key;
|
||||
if (value && typeof value === 'object' && !Array.isArray(value)) {
|
||||
Object.assign(results, flattenObject(value, newKey));
|
||||
} else {
|
||||
results[newKey] = String(value);
|
||||
}
|
||||
}
|
||||
return results;
|
||||
};
|
||||
|
||||
export default function UpdateView({ fileNode, folders: availablefolders }: { fileNode: any; folders: any[]; }) {
|
||||
const router = useRouter();
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [isExtracting, setIsExtracting] = useState(false);
|
||||
|
||||
// 1. Initialize Basic Info
|
||||
const [name, setName] = useState(fileNode.name);
|
||||
const [description, setDescription] = useState(fileNode.description || "");
|
||||
const [parentId, setParentId] = useState(fileNode.parentId || "");
|
||||
|
||||
// 2. Initialize Metadata from DB (all checked by default)
|
||||
const initialMetadata: MetadataPair[] = Object.entries(fileNode.metadata || {})
|
||||
.filter(([key]) => !['type', 'mimeType'].includes(key))
|
||||
.filter(([key]) => !['type', 'mimeType', 'magicFilled', 'details'].includes(key))
|
||||
.map(([key, value]) => ({
|
||||
key,
|
||||
value: String(value),
|
||||
|
|
@ -54,23 +61,23 @@ export default function UpdateView({
|
|||
|
||||
const [customMetadata, setCustomMetadata] = useState<MetadataPair[]>(initialMetadata);
|
||||
|
||||
// --- MAGIC FILL LOGIC ---
|
||||
const handleMagicEnhance = async () => {
|
||||
setIsExtracting(true);
|
||||
try {
|
||||
const result = await getMetadataPreviewAction(fileNode.id);
|
||||
if (result.success) {
|
||||
// Convert extracted JSON into pending rows
|
||||
const extractedRows: MetadataPair[] = Object.entries(result.data ?? {})
|
||||
.filter(([key]) => !['type', 'mimeType'].includes(key))
|
||||
// Flatten the nested 'details' and top level props
|
||||
const flatData = flattenObject(result.data);
|
||||
|
||||
const extractedRows: MetadataPair[] = Object.entries(flatData)
|
||||
.filter(([key]) => !['type', 'mimeType', 'title'].includes(key) && !key.includes('Binary Data'))
|
||||
.map(([key, value]) => ({
|
||||
key,
|
||||
value: String(value),
|
||||
selected: true, // Default to checked as requested
|
||||
selected: true,
|
||||
isPending: true
|
||||
}));
|
||||
|
||||
// Merge logic: Add only if the key doesn't already exist in our list
|
||||
setCustomMetadata(prev => {
|
||||
const existingKeys = new Set(prev.map(r => r.key));
|
||||
const filteredNew = extractedRows.filter(r => !existingKeys.has(r.key));
|
||||
|
|
@ -78,7 +85,7 @@ export default function UpdateView({
|
|||
});
|
||||
}
|
||||
} catch (err) {
|
||||
alert("Failed to extract metadata. Ensure service is configured correctly.");
|
||||
alert("Failed to extract metadata.");
|
||||
} finally {
|
||||
setIsExtracting(false);
|
||||
}
|
||||
|
|
@ -92,7 +99,6 @@ export default function UpdateView({
|
|||
formData.append("description", description);
|
||||
formData.append("parentId", parentId);
|
||||
|
||||
// Convert array back to object, ONLY including selected/checked rows
|
||||
const metadataObj = customMetadata.reduce((acc, curr) => {
|
||||
if (curr.selected && curr.key.trim()) {
|
||||
acc[curr.key.trim()] = curr.value;
|
||||
|
|
@ -114,131 +120,77 @@ export default function UpdateView({
|
|||
|
||||
return (
|
||||
<Paper sx={{ p: { xs: 3, md: 6 }, borderRadius: 4 }} elevation={3}>
|
||||
<Button startIcon={<ArrowBackIcon />} onClick={() => router.back()} sx={{ mb: 2 }}>
|
||||
Back
|
||||
</Button>
|
||||
<Button startIcon={<ArrowBackIcon />} onClick={() => router.back()} sx={{ mb: 2 }}>Back</Button>
|
||||
|
||||
<Typography variant="h4" fontWeight={900} color="primary" gutterBottom>
|
||||
Edit File Details
|
||||
</Typography>
|
||||
<Typography variant="h4" fontWeight={900} color="primary" gutterBottom>Edit File Details</Typography>
|
||||
|
||||
{/* --- MAGIC FILL BUTTON --- */}
|
||||
<Box sx={{ mb: 4, p: 2, bgcolor: '#f0f7ff', borderRadius: 2, border: '1px dashed #1976d2' }}>
|
||||
<Stack direction="row" justifyContent="space-between" alignItems="center">
|
||||
<Box>
|
||||
<Typography variant="subtitle1" fontWeight="bold">Enrich Metadata</Typography>
|
||||
<Typography variant="caption" color="text.secondary">
|
||||
Extract tags like GPS, Author, and Dimensions from the original file.
|
||||
</Typography>
|
||||
<Typography variant="caption" color="text.secondary">Extract GPS, Camera Specs, and Dimensions.</Typography>
|
||||
</Box>
|
||||
<Button
|
||||
variant="contained"
|
||||
color="secondary"
|
||||
variant="contained" color="secondary"
|
||||
startIcon={isExtracting ? <CircularProgress size={20} color="inherit" /> : <AutoFixHighIcon />}
|
||||
onClick={handleMagicEnhance}
|
||||
disabled={isExtracting}
|
||||
onClick={handleMagicEnhance} disabled={isExtracting}
|
||||
>
|
||||
{isExtracting ? 'Extracting...' : 'Magic Fill'}
|
||||
</Button>
|
||||
</Stack>
|
||||
</Box>
|
||||
|
||||
<Stack spacing={4} sx={{ mt: 2 }}>
|
||||
<TextField
|
||||
label="File Name"
|
||||
fullWidth value={name}
|
||||
onChange={(e) => setName(e.target.value)}
|
||||
slotProps={{ inputLabel: { shrink: true } }}
|
||||
/>
|
||||
<Stack spacing={4}>
|
||||
<TextField label="File Name" fullWidth value={name} onChange={(e) => setName(e.target.value)} slotProps={{ inputLabel: { shrink: true } }} />
|
||||
|
||||
<TextField
|
||||
id="update-dest-select"
|
||||
select fullWidth label="Destination Folder"
|
||||
value={parentId}
|
||||
onChange={(e) => setParentId(e.target.value)}
|
||||
slotProps={{
|
||||
select: { displayEmpty: true },
|
||||
inputLabel: { shrink: true }
|
||||
}}
|
||||
>
|
||||
<TextField select fullWidth label="Destination" value={parentId} onChange={(e) => setParentId(e.target.value)}>
|
||||
<MenuItem value=""><em>-- Root --</em></MenuItem>
|
||||
{availablefolders?.map((f: any) => (
|
||||
<MenuItem key={f.id} value={f.id}>{f.name}</MenuItem>
|
||||
))}
|
||||
{availablefolders?.map((f: any) => (<MenuItem key={f.id} value={f.id}>{f.name}</MenuItem>))}
|
||||
</TextField>
|
||||
|
||||
<Box>
|
||||
<Box sx={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center', mb: 2 }}>
|
||||
<Typography variant="h6" fontWeight="700" sx={{ display: 'flex', alignItems: 'center', gap: 1 }}>
|
||||
<AssignmentIcon color="primary" /> Metadata Attributes
|
||||
</Typography>
|
||||
<Button
|
||||
startIcon={<AddCircleOutlineIcon />}
|
||||
size="small"
|
||||
onClick={() => setCustomMetadata([...customMetadata, { key: "", value: "", selected: true }])}
|
||||
>
|
||||
Add Field
|
||||
</Button>
|
||||
</Box>
|
||||
<Stack direction="row" justifyContent="space-between" mb={2}>
|
||||
<Typography variant="h6" fontWeight="700"><AssignmentIcon /> Attributes</Typography>
|
||||
<Button startIcon={<AddCircleOutlineIcon />} onClick={() => setCustomMetadata([...customMetadata, { key: "", value: "", selected: true }])}>Add Field</Button>
|
||||
</Stack>
|
||||
|
||||
<Stack spacing={2}>
|
||||
{customMetadata.map((row, index) => (
|
||||
<Box key={index}>
|
||||
<Grid container spacing={1} alignItems="center">
|
||||
<Grid size={{ xs: 1 }}>
|
||||
<Tooltip title={row.selected ? "Save this field" : "Ignore this field"}>
|
||||
<Checkbox
|
||||
checked={row.selected}
|
||||
onChange={(e) => {
|
||||
const updated = [...customMetadata];
|
||||
updated[index].selected = e.target.checked;
|
||||
setCustomMetadata(updated);
|
||||
}}
|
||||
/>
|
||||
</Tooltip>
|
||||
<Grid item xs={1}>
|
||||
<Checkbox checked={row.selected} onChange={(e) => {
|
||||
const updated = [...customMetadata];
|
||||
updated[index].selected = e.target.checked;
|
||||
setCustomMetadata(updated);
|
||||
}} />
|
||||
</Grid>
|
||||
<Grid size={{ xs: 4 }}>
|
||||
<TextField
|
||||
fullWidth size="small" placeholder="Key"
|
||||
value={row.key}
|
||||
disabled={row.isPending} // Usually best to keep extracted keys as-is
|
||||
sx={{ bgcolor: row.isPending ? '#e8f5e9' : 'transparent' }}
|
||||
<Grid item xs={4}>
|
||||
<TextField fullWidth size="small" value={row.key} disabled={row.isPending} sx={{ bgcolor: row.isPending ? '#e8f5e9' : 'transparent' }}
|
||||
onChange={(e) => {
|
||||
const updated = [...customMetadata];
|
||||
updated[index].key = e.target.value;
|
||||
setCustomMetadata(updated);
|
||||
}}
|
||||
/>
|
||||
}} />
|
||||
</Grid>
|
||||
<Grid size={{ xs: 6 }}>
|
||||
<TextField
|
||||
fullWidth size="small" placeholder="Value"
|
||||
value={row.value}
|
||||
sx={{ bgcolor: row.isPending ? '#e8f5e9' : 'transparent' }}
|
||||
<Grid item xs={6}>
|
||||
<TextField fullWidth size="small" value={row.value} sx={{ bgcolor: row.isPending ? '#e8f5e9' : 'transparent' }}
|
||||
onChange={(e) => {
|
||||
const updated = [...customMetadata];
|
||||
updated[index].value = e.target.value;
|
||||
setCustomMetadata(updated);
|
||||
}}
|
||||
/>
|
||||
}} />
|
||||
</Grid>
|
||||
<Grid size={{ xs: 1 }}>
|
||||
<IconButton color="error" onClick={() => setCustomMetadata(customMetadata.filter((_, i) => i !== index))}>
|
||||
<DeleteOutlineIcon />
|
||||
</IconButton>
|
||||
<Grid item xs={1}>
|
||||
<IconButton color="error" onClick={() => setCustomMetadata(customMetadata.filter((_, i) => i !== index))}><DeleteOutlineIcon /></IconButton>
|
||||
</Grid>
|
||||
</Grid>
|
||||
|
||||
{/* --- GOOGLE MAPS SHORTCUT --- */}
|
||||
{row.key.toLowerCase().includes('latitude') && row.value && (
|
||||
<Box sx={{ ml: 6, mt: 0.5 }}>
|
||||
<Button
|
||||
size="small"
|
||||
startIcon={<MapIcon />}
|
||||
href={`https://www.google.com/maps?q=${row.value},${customMetadata.find(m => m.key.toLowerCase().includes('longitude'))?.value}`}
|
||||
target="_blank"
|
||||
>
|
||||
Verify GPS on Map
|
||||
<Box sx={{ ml: 7, mt: 0.5 }}>
|
||||
<Button size="small" startIcon={<MapIcon />} target="_blank"
|
||||
href={`https://www.google.com/maps?q=${row.value},${customMetadata.find(m => m.key.toLowerCase().includes('longitude'))?.value}`}>
|
||||
View on Map
|
||||
</Button>
|
||||
</Box>
|
||||
)}
|
||||
|
|
@ -247,21 +199,9 @@ export default function UpdateView({
|
|||
</Stack>
|
||||
</Box>
|
||||
|
||||
<TextField
|
||||
label="Description"
|
||||
multiline rows={4} fullWidth
|
||||
value={description}
|
||||
onChange={(e) => setDescription(e.target.value)}
|
||||
slotProps={{ inputLabel: { shrink: true } }}
|
||||
/>
|
||||
<TextField label="Description" multiline rows={3} fullWidth value={description} onChange={(e) => setDescription(e.target.value)} />
|
||||
|
||||
<Button
|
||||
variant="contained" size="large" fullWidth
|
||||
startIcon={<SaveIcon />}
|
||||
onClick={handleUpdate}
|
||||
disabled={loading}
|
||||
sx={{ py: 1.5, fontWeight: 'bold' }}
|
||||
>
|
||||
<Button variant="contained" size="large" fullWidth startIcon={<SaveIcon />} onClick={handleUpdate} disabled={loading}>
|
||||
{loading ? "Saving..." : "Save Changes"}
|
||||
</Button>
|
||||
</Stack>
|
||||
|
|
|
|||
83
src/lib/metadata-extractor-old-2.ts
Normal file
83
src/lib/metadata-extractor-old-2.ts
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
// src/lib/metadata-extractor.ts
|
||||
// 1. Update your import to use the new named export
|
||||
//import * as PDFLib from 'pdf-parse';
|
||||
//import { PDFParse } from 'pdf-parse';
|
||||
//import {pdf} from 'pdf-parse';
|
||||
import * as PdfParse from 'pdf-parse-new';
|
||||
import sharp from 'sharp';
|
||||
import exifReader from 'exif-reader';
|
||||
|
||||
/**
|
||||
* Converts EXIF DMS (Degrees, Minutes, Seconds) array to Decimal Degrees.
|
||||
*/
|
||||
function convertDMSToDD(dms: any, ref: string): string {
|
||||
if (!Array.isArray(dms) || dms.length < 3) return String(dms);
|
||||
|
||||
const [degrees, minutes, seconds] = dms;
|
||||
let dd = degrees + (minutes / 60) + (seconds / 3600);
|
||||
|
||||
if (ref === 'S' || ref === 'W') {
|
||||
dd = dd * -1;
|
||||
}
|
||||
return dd.toFixed(6);
|
||||
}
|
||||
|
||||
/**
|
||||
* RECURSIVE SANITIZER:
|
||||
* Converts Buffers to strings, standardizes keys, and handles GPS conversion.
|
||||
*/
|
||||
function sanitizeMetadata(obj: any): any {
|
||||
if (obj === null || typeof obj !== 'object') return obj;
|
||||
if (obj instanceof Date) return obj.toISOString();
|
||||
if (Buffer.isBuffer(obj)) return `[Binary Data: ${obj.length} bytes]`;
|
||||
if (Array.isArray(obj)) return obj.map(sanitizeMetadata);
|
||||
|
||||
const sanitized: any = {};
|
||||
for (const [key, value] of Object.entries(obj)) {
|
||||
const cleanKey = key.charAt(0).toLowerCase() + key.slice(1);
|
||||
|
||||
// Specifically handle GPS Latitude/Longitude Arrays
|
||||
if (cleanKey === 'gPSLatitude' && obj['gPSLatitudeRef']) {
|
||||
sanitized[cleanKey] = convertDMSToDD(value, obj['gPSLatitudeRef']);
|
||||
continue;
|
||||
}
|
||||
if (cleanKey === 'gPSLongitude' && obj['gPSLongitudeRef']) {
|
||||
sanitized[cleanKey] = convertDMSToDD(value, obj['gPSLongitudeRef']);
|
||||
continue;
|
||||
}
|
||||
|
||||
sanitized[cleanKey] = sanitizeMetadata(value);
|
||||
}
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
export async function extractMetadata(buffer: Buffer, filename: string): Promise<any> {
|
||||
const extension = filename.split('.').pop()?.toLowerCase();
|
||||
|
||||
if (extension === 'pdf') {
|
||||
// Create parser instance
|
||||
const parser = new PdfParse.SmartPDFParser({
|
||||
oversaturationFactor: 2.0,
|
||||
enableFastPath: true
|
||||
});
|
||||
console.log(`--- PDF Debug Start: ${filename} ---`);
|
||||
const result = await parser.parse(buffer);
|
||||
console.log(`Parsed ${result.numpages} pages using ${result._meta.method}`);
|
||||
console.log(`Parsed ${result.info} info using ${result._meta.method}`);
|
||||
console.log(`Parsed ${result.info} info using ${result._meta.method}`);
|
||||
console.log(JSON.stringify(result.info, null, 2));
|
||||
//const { text, numpages, info } = await pdf(buffer);
|
||||
//const parser = new PDFParse(buffer);
|
||||
// `text` → full document text
|
||||
// `numpages` → page count
|
||||
// `info` → metadata (author, creation date, etc.)
|
||||
|
||||
// console.log(`Pages: ${numpages}`);
|
||||
// console.log(`Author: ${info.Author}`);
|
||||
// console.log(text.slice(0, 200)); // preview first 200 chars
|
||||
return {
|
||||
type: 'PDF',
|
||||
title: filename
|
||||
};
|
||||
}
|
||||
}
|
||||
145
src/lib/metadata-extractor-old.ts
Normal file
145
src/lib/metadata-extractor-old.ts
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
// src/lib/metadata-extractor.ts
|
||||
// 1. Update your import to use the new named export
|
||||
import * as PDFLib from 'pdf-parse';
|
||||
import sharp from 'sharp';
|
||||
import exifReader from 'exif-reader';
|
||||
|
||||
/**
|
||||
* Converts EXIF DMS (Degrees, Minutes, Seconds) array to Decimal Degrees.
|
||||
*/
|
||||
function convertDMSToDD(dms: any, ref: string): string {
|
||||
if (!Array.isArray(dms) || dms.length < 3) return String(dms);
|
||||
|
||||
const [degrees, minutes, seconds] = dms;
|
||||
let dd = degrees + (minutes / 60) + (seconds / 3600);
|
||||
|
||||
if (ref === 'S' || ref === 'W') {
|
||||
dd = dd * -1;
|
||||
}
|
||||
return dd.toFixed(6);
|
||||
}
|
||||
|
||||
/**
|
||||
* RECURSIVE SANITIZER:
|
||||
* Converts Buffers to strings, standardizes keys, and handles GPS conversion.
|
||||
*/
|
||||
function sanitizeMetadata(obj: any): any {
|
||||
if (obj === null || typeof obj !== 'object') return obj;
|
||||
if (obj instanceof Date) return obj.toISOString();
|
||||
if (Buffer.isBuffer(obj)) return `[Binary Data: ${obj.length} bytes]`;
|
||||
if (Array.isArray(obj)) return obj.map(sanitizeMetadata);
|
||||
|
||||
const sanitized: any = {};
|
||||
for (const [key, value] of Object.entries(obj)) {
|
||||
const cleanKey = key.charAt(0).toLowerCase() + key.slice(1);
|
||||
|
||||
// Specifically handle GPS Latitude/Longitude Arrays
|
||||
if (cleanKey === 'gPSLatitude' && obj['gPSLatitudeRef']) {
|
||||
sanitized[cleanKey] = convertDMSToDD(value, obj['gPSLatitudeRef']);
|
||||
continue;
|
||||
}
|
||||
if (cleanKey === 'gPSLongitude' && obj['gPSLongitudeRef']) {
|
||||
sanitized[cleanKey] = convertDMSToDD(value, obj['gPSLongitudeRef']);
|
||||
continue;
|
||||
}
|
||||
|
||||
sanitized[cleanKey] = sanitizeMetadata(value);
|
||||
}
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
export async function extractMetadata(buffer: Buffer, filename: string): Promise<any> {
|
||||
const extension = filename.split('.').pop()?.toLowerCase();
|
||||
|
||||
if (extension === 'pdf') {
|
||||
console.log(`--- PDF Debug Start: ${filename} ---`);
|
||||
|
||||
// Initialize using the data property for our Buffer
|
||||
const parser = new PDFLib.PDFParse({
|
||||
data: buffer,
|
||||
// disableWorker: true, // Crucial for Next.js
|
||||
// verbosity: 0
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await parser.getInfo({ parsePageInfo: true });
|
||||
|
||||
// LOGGING TO TERMINAL
|
||||
console.log(`✅ Total pages: ${result.total}`);
|
||||
console.log(`✅ Title: ${result.info?.Title}`);
|
||||
console.log(`✅ Author: ${result.info?.Author}`);
|
||||
console.log(`✅ Creator: ${result.info?.Creator}`);
|
||||
|
||||
// Date info
|
||||
const dates = result.getDateNode();
|
||||
console.log(`✅ Creation Date: ${dates.CreationDate}`);
|
||||
|
||||
console.log('--- PDF Debug End ---');
|
||||
}
|
||||
// return {
|
||||
// type: 'PDF',
|
||||
// title: filename,
|
||||
// pageCount: result.total,
|
||||
// details: result.info, // This sends the raw info object back
|
||||
// };
|
||||
// } catch (err: any) {
|
||||
// console.error("❌ PDF Parsing Error inside debug block:", err.message);
|
||||
// return { type: 'PDF', error: err.message };
|
||||
// } finally {
|
||||
// await parser.destroy();
|
||||
// }
|
||||
|
||||
// --- 1. PDF EXTRACTION (v2 Class-based) ---
|
||||
// if (extension === 'pdf') {
|
||||
// // Access PDFParse from the namespace
|
||||
// const parser = new PDFLib.PDFParse({
|
||||
// data: buffer,
|
||||
// disableWorker: true, // Prevents the .mjs worker error
|
||||
// verbosity: 0
|
||||
// });
|
||||
|
||||
// try {
|
||||
// const infoResult = await parser.getInfo({ parsePageInfo: true });
|
||||
// const textResult = await parser.getText();
|
||||
|
||||
// return {
|
||||
// type: 'PDF',
|
||||
// title: filename,
|
||||
// details: sanitizeMetadata(infoResult.info || {}),
|
||||
// pageCount: infoResult.total || 0,
|
||||
// textPreview: textResult.text ? textResult.text.substring(0, 200).replace(/\s+/g, ' ') : ""
|
||||
// };
|
||||
// } finally {
|
||||
// // Free memory
|
||||
// await parser.destroy();
|
||||
// }
|
||||
// }
|
||||
|
||||
if (['jpg', 'jpeg', 'png', 'webp'].includes(extension || '')) {
|
||||
const image = sharp(buffer);
|
||||
const metadata = await image.metadata();
|
||||
|
||||
let details = {};
|
||||
if (metadata.exif) {
|
||||
try {
|
||||
const rawExif = exifReader(metadata.exif);
|
||||
details = sanitizeMetadata(rawExif);
|
||||
} catch (e) {
|
||||
console.warn("EXIF Parse failed");
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
type: `IMAGE (${metadata.format?.toUpperCase()})`,
|
||||
dimensions: `${metadata.width}x${metadata.height}`,
|
||||
title: filename,
|
||||
details: details
|
||||
};
|
||||
}
|
||||
|
||||
return { type: 'FILE', title: filename };
|
||||
} catch (error) {
|
||||
console.error(`Extraction failed: ${filename}`, error);
|
||||
return { type: 'FILE', title: filename, error: "Extraction failed" };
|
||||
}
|
||||
}
|
||||
|
|
@ -1,100 +1,108 @@
|
|||
import * as pdf from 'pdf-parse';
|
||||
import EPub from 'epub';
|
||||
// src/lib/metadata-extractor.ts
|
||||
// 1. Update your import to use the new named export
|
||||
//import * as PDFLib from 'pdf-parse';
|
||||
//import { PDFParse } from 'pdf-parse';
|
||||
//import {pdf} from 'pdf-parse';
|
||||
import * as PdfParse from 'pdf-parse-new';
|
||||
import sharp from 'sharp';
|
||||
import exifReader from 'exif-reader';
|
||||
|
||||
export interface ExtractedMetadata {
|
||||
title?: string;
|
||||
author?: string;
|
||||
subject?: string;
|
||||
dimensions?: string;
|
||||
pageCount?: number;
|
||||
latitude?: number;
|
||||
longitude?: number;
|
||||
deviceModel?: string | null;
|
||||
exposureTime?: string | null;
|
||||
fNumber?: number | null;
|
||||
iso?: number | null;
|
||||
type: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper: Converts [Degrees, Minutes, Seconds] to Decimal
|
||||
* Converts EXIF DMS (Degrees, Minutes, Seconds) array to Decimal Degrees.
|
||||
*/
|
||||
function parseGps(values: number[] | undefined, ref: string | undefined): number | null {
|
||||
if (!values || values.length < 3) return null;
|
||||
const decimal = values[0] + values[1] / 60 + values[2] / 3600;
|
||||
const multiplier = (ref === 'S' || ref === 'W') ? -1 : 1;
|
||||
return Number((decimal * multiplier).toFixed(6));
|
||||
}
|
||||
function convertDMSToDD(dms: any, ref: string): string {
|
||||
if (!Array.isArray(dms) || dms.length < 3) return String(dms);
|
||||
|
||||
/**
|
||||
* Helper: Converts Rational fractions (1/4000) or Objects to readable values
|
||||
*/
|
||||
function parseRational(value: any, asString = false): string | number | null {
|
||||
if (!value) return null;
|
||||
if (typeof value === 'number') return value;
|
||||
if (value.numerator && value.denominator) {
|
||||
if (asString) return `${value.numerator}/${value.denominator}`;
|
||||
return value.numerator / value.denominator;
|
||||
const [degrees, minutes, seconds] = dms;
|
||||
let dd = degrees + (minutes / 60) + (seconds / 3600);
|
||||
|
||||
if (ref === 'S' || ref === 'W') {
|
||||
dd = dd * -1;
|
||||
}
|
||||
return null;
|
||||
return dd.toFixed(6);
|
||||
}
|
||||
|
||||
export async function extractMetadata(buffer: Buffer, filename: string): Promise<ExtractedMetadata> {
|
||||
/**
|
||||
* RECURSIVE SANITIZER:
|
||||
* Converts Buffers to strings, standardizes keys, and handles GPS conversion.
|
||||
*/
|
||||
function sanitizeMetadata(obj: any): any {
|
||||
if (obj === null || typeof obj !== 'object') return obj;
|
||||
if (obj instanceof Date) return obj.toISOString();
|
||||
if (Buffer.isBuffer(obj)) return `[Binary Data: ${obj.length} bytes]`;
|
||||
if (Array.isArray(obj)) return obj.map(sanitizeMetadata);
|
||||
|
||||
const sanitized: any = {};
|
||||
for (const [key, value] of Object.entries(obj)) {
|
||||
const cleanKey = key.charAt(0).toLowerCase() + key.slice(1);
|
||||
|
||||
// Specifically handle GPS Latitude/Longitude Arrays
|
||||
if (cleanKey === 'gPSLatitude' && obj['gPSLatitudeRef']) {
|
||||
sanitized[cleanKey] = convertDMSToDD(value, obj['gPSLatitudeRef']);
|
||||
continue;
|
||||
}
|
||||
if (cleanKey === 'gPSLongitude' && obj['gPSLongitudeRef']) {
|
||||
sanitized[cleanKey] = convertDMSToDD(value, obj['gPSLongitudeRef']);
|
||||
continue;
|
||||
}
|
||||
|
||||
sanitized[cleanKey] = sanitizeMetadata(value);
|
||||
}
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
export async function extractMetadata(buffer: Buffer, filename: string): Promise<any> {
|
||||
const extension = filename.split('.').pop()?.toLowerCase();
|
||||
|
||||
try {
|
||||
// --- 1. PDF EXTRACTION ---
|
||||
// --- 1. PDF EXTRACTION (pdf-parse-new) ---
|
||||
if (extension === 'pdf') {
|
||||
const parsePdf = (pdf as any).default || pdf;
|
||||
const data = await parsePdf(buffer);
|
||||
const parser = new PdfParse.SmartPDFParser({
|
||||
oversaturationFactor: 2.0,
|
||||
enableFastPath: true
|
||||
});
|
||||
|
||||
console.log(`--- PDF Extraction Start: ${filename} ---`);
|
||||
const result = await parser.parse(buffer);
|
||||
|
||||
return {
|
||||
type: 'PDF',
|
||||
title: data.info?.Title || filename,
|
||||
author: data.info?.Author,
|
||||
subject: data.info?.Subject,
|
||||
pageCount: data.numpages,
|
||||
title: filename,
|
||||
// pageCount is specifically named 'numpages' in this library
|
||||
pageCount: result.numpages || 0,
|
||||
// We sanitize the result.info to handle those nested Custom objects
|
||||
details: sanitizeMetadata(result.info || {}),
|
||||
// Use result.text for a preview if available
|
||||
textPreview: result.text ? result.text.substring(0, 200).replace(/\s+/g, ' ') : ""
|
||||
};
|
||||
}
|
||||
|
||||
// --- 2. IMAGE EXTRACTION (Enhanced) ---
|
||||
// --- 2. IMAGE EXTRACTION ---
|
||||
if (['jpg', 'jpeg', 'png', 'webp'].includes(extension || '')) {
|
||||
const image = sharp(buffer);
|
||||
const metadata = await image.metadata();
|
||||
|
||||
let exifData: Partial<ExtractedMetadata> = {};
|
||||
|
||||
let details = {};
|
||||
if (metadata.exif) {
|
||||
try {
|
||||
const exif = exifReader(metadata.exif) as any;
|
||||
console.log(" exif ");
|
||||
console.log(exif);
|
||||
exifData = {
|
||||
deviceModel: exif.image?.Model || null,
|
||||
exposureTime: parseRational(exif.photo?.ExposureTime, true) as string,
|
||||
fNumber: parseRational(exif.photo?.FNumber) as number,
|
||||
iso: exif.photo?.ISOSpeedRatings || null,
|
||||
latitude: parseGps(exif.gps?.GPSLatitude, exif.gps?.GPSLatitudeRef) || undefined,
|
||||
longitude: parseGps(exif.gps?.GPSLongitude, exif.gps?.GPSLongitudeRef) || undefined,
|
||||
};
|
||||
} catch (exifError) {
|
||||
console.warn("Could not parse EXIF data for:", filename, exifError);
|
||||
const rawExif = exifReader(metadata.exif);
|
||||
details = sanitizeMetadata(rawExif);
|
||||
} catch (e) {
|
||||
console.warn("EXIF Parse failed");
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
type: `IMAGE (${metadata.format?.toUpperCase()})`,
|
||||
dimensions: metadata.width && metadata.height ? `${metadata.width}x${metadata.height}` : undefined,
|
||||
dimensions: `${metadata.width}x${metadata.height}`,
|
||||
title: filename,
|
||||
...exifData
|
||||
details: details
|
||||
};
|
||||
}
|
||||
|
||||
// Default Fallback
|
||||
return { type: 'FILE', title: filename };
|
||||
} catch (error) {
|
||||
console.error(`Extraction failed for ${filename}:`, error);
|
||||
return { type: 'FILE', title: filename };
|
||||
} catch (error: any) {
|
||||
console.error(`❌ Extraction failed for ${filename}:`, error.message);
|
||||
return { type: 'FILE', title: filename, error: error.message };
|
||||
}
|
||||
}
|
||||
BIN
src/lib/text.pdf
Normal file
BIN
src/lib/text.pdf
Normal file
Binary file not shown.
59
src/lib/transformers.ts
Normal file
59
src/lib/transformers.ts
Normal file
|
|
@ -0,0 +1,59 @@
|
|||
/**
|
||||
* Helper: Converts [Degrees, Minutes, Seconds] to Decimal Degrees
|
||||
*/
|
||||
const convertDMSToDD = (dms: number[] | undefined, ref: string | undefined): number | null => {
|
||||
if (!dms || dms.length < 3) return null;
|
||||
const [degrees, minutes, seconds] = dms;
|
||||
let dd = degrees + (minutes / 60) + (seconds / 3600);
|
||||
|
||||
if (ref === 'S' || ref === 'W') {
|
||||
dd = dd * -1;
|
||||
}
|
||||
return parseFloat(dd.toFixed(6));
|
||||
};
|
||||
|
||||
/**
|
||||
* MAPPER: Transforms the raw database JSON into a structured GUI object.
|
||||
* Designed to handle the lowercase keys generated by the sanitizer.
|
||||
*/
|
||||
export const mapImageMetadata = (metadata: any) => {
|
||||
if (!metadata || !metadata.details) {
|
||||
return { title: metadata?.title || "Unknown File" };
|
||||
}
|
||||
|
||||
const { details, dimensions, title } = metadata;
|
||||
const { image = {}, photo = {}, gpsInfo = {} } = details;
|
||||
|
||||
return {
|
||||
fileName: title,
|
||||
device: `${image.make || ''} ${image.model || ''}`.trim() || 'Unknown Device',
|
||||
|
||||
// Dates from EXIF are strings after sanitization
|
||||
timestamp: photo.dateTimeOriginal ? new Date(photo.dateTimeOriginal) : null,
|
||||
|
||||
resolution: {
|
||||
width: photo.pixelXDimension || dimensions?.split('x')[0],
|
||||
height: photo.pixelYDimension || dimensions?.split('x')[1],
|
||||
},
|
||||
|
||||
settings: {
|
||||
aperture: photo.fNumber ? `f/${photo.fNumber}` : 'N/A',
|
||||
shutterSpeed: photo.exposureTime
|
||||
? (photo.exposureTime < 1
|
||||
? `1/${Math.round(1 / photo.exposureTime)}s`
|
||||
: `${photo.exposureTime}s`)
|
||||
: 'N/A',
|
||||
iso: photo.iSOSpeedRatings || 'N/A',
|
||||
focalLength: photo.focalLength ? `${photo.focalLength}mm` : 'N/A',
|
||||
},
|
||||
|
||||
location: {
|
||||
latitude: convertDMSToDD(gpsInfo.gpsLatitude, gpsInfo.gpsLatitudeRef),
|
||||
longitude: convertDMSToDD(gpsInfo.gpsLongitude, gpsInfo.gpsLongitudeRef),
|
||||
altitude: gpsInfo.gpsAltitude ? Math.round(gpsInfo.gpsAltitude) : null,
|
||||
mapUrl: (gpsInfo.gpsLatitude && gpsInfo.gpsLongitude)
|
||||
? `https://www.google.com/maps?q=${convertDMSToDD(gpsInfo.gpsLatitude, gpsInfo.gpsLatitudeRef)},${convertDMSToDD(gpsInfo.gpsLongitude, gpsInfo.gpsLongitudeRef)}`
|
||||
: null
|
||||
}
|
||||
};
|
||||
};
|
||||
|
|
@ -1,6 +1,5 @@
|
|||
// src/services/metadata-service.ts
|
||||
import { getOneDriveFileBuffer } from "./onedrive";
|
||||
// 1. Change the import name
|
||||
import { extractMetadata } from "@/lib/metadata-extractor";
|
||||
import { getFileNodeById, updateFileNode } from "@/data-access/file-nodes";
|
||||
|
||||
|
|
@ -8,16 +7,19 @@ export async function enrichFileMetadata(fileId: string, token: string) {
|
|||
const node = await getFileNodeById(fileId);
|
||||
if (!node || !node.oneDriveId) throw new Error("Node not found");
|
||||
|
||||
// Fetch file from OneDrive
|
||||
const buffer = await getOneDriveFileBuffer(node.oneDriveId, token);
|
||||
|
||||
// 2. Use the standard extractor (it handles PDF, Image, etc. automatically)
|
||||
// Extract deep metadata (PDF, JPEG, etc.)
|
||||
const deepMetadata = await extractMetadata(buffer, node.name);
|
||||
|
||||
// Merge with existing metadata
|
||||
const updatedMetadata = {
|
||||
...(node.metadata as object),
|
||||
...deepMetadata, // This now contains type, dimensions, gps, deviceModel, etc.
|
||||
...deepMetadata,
|
||||
magicFilled: true
|
||||
};
|
||||
|
||||
// Update DB
|
||||
return await updateFileNode(fileId, { metadata: updatedMetadata });
|
||||
}
|
||||
Loading…
Reference in a new issue