2021112-Upload-Large-File

upload api


import { connectToDb, fileExists } from "@/lib/mongodb";

import { NextResponse } from "next/server";

import { Readable } from "stream";

import formidable, { errors as formidableErrors } from 'formidable';

var fs = require('fs');

var md5 = require('md5');

  

export const config = {

api: {

bodyParser: false,

}

};

type ProcessedFiles = Array<[string, File]>;

export default async function handler(req: any, res: any) {

const { bucket } = await connectToDb();

// get the form data

// const data = await req.formData();

// Access uploaded files directly using req.files

//const files = Array.from(req.files.entries());

  

let status = 200,

resultBody = { status: 'ok', message: 'Files were uploaded successfully' };

  

const form = formidable({ uploadDir: "/tmp" });

let fields;

let files;

try {

form.parse(req, async (err, fields, files) => {

console.log('fields:', fields);

console.log('files:', files);

for (const [key, value] of Object.entries(files)) {

const isFile = typeof value == "object";

if (isFile) {

let file: any = value[0]

let filename = file.originalFilename

let type = file.mimetype

let buffer = fs.readFileSync(file.filepath);

const stream = Readable.from(buffer);

const hash = md5(buffer)

const existing = await fileExists(hash);

if (existing) {

// If file already exists, let's skip it.

// If you want a different behavior such as override, modify this part.

continue;

}

const uploadStream = bucket.openUploadStream(filename, {

// make sure to add content type so that it will be easier to set later.

contentType: type,

metadata: {

hash:hash

}, //add your metadata here if any

});

  

// pipe the readable stream to a writeable stream to save it to the database

await stream.pipe(uploadStream);

res.status(200).json({

hash: hash

})

}

  

}

});

  
  

} catch (err: any) {

// example to check for a very specific error

if (err.code === formidableErrors.maxFieldsExceeded) {

  

}

console.error(err);

res.writeHead(err.httpCode || 400, { 'Content-Type': 'text/plain' });

res.end(String(err));

return;

}

  

return res.json({ success: true });

}

response image

import { MongoClient, ObjectId, GridFSBucket } from 'mongodb';

import clientPromise, { connectToDb } from "@/lib/mongodb";


export default async function handler(req:any, res:any) {

const client = await clientPromise;

const {hash} = req.query;

const db = client.db("fanstick");

const metadata:any = await db

.collection('media.files')

.find({"metadata.hash":hash})

.toArray()

console.log(metadata)

const {bucket}= await connectToDb()

res.writeHead(200, { 'Content-Type': metadata[0].contentType });

bucket.openDownloadStream(metadata[0]._id)

.on('data', (chunk) => {

res.write(chunk);

})

.on('end', () => {

res.end();

})

.on('error', (err) => {

res.status(500).json({ error: err.message });

});

// const file:any = await db

// .collection('media.chunks')

// .find({"files_id":metadata[0]._id})

// .toArray()

// console.log(file[0].data)

// res.setHeader('Content-Type', metadata[0].contentType)

// res.send(file[0].data)

}

Frontend

fetch(`/backend/upload/file`, {

method: "POST",

body: formData

}).then((res) => res.json())

.then(({hash}) => {

const image_url = `${process.env.Deploy_URL}/api/file/${hash}`

setImage(image_url)

form.setFieldValue(field.name,image_url)

})

Ref

80分鐘快速了解大型語言模型-5-30有咒術迴戰雷

https://i.imgur.com/U6OeFM7.png 客製化GPT https://openai.com/blog/introducing-gpts 串接dalle 自動生成頭像

https://i.imgur.com/oBxJeFw.png GPT: Generative Pre-trained Transformer others Google Bard anthropic Claude GPT 訓練:文字接龍 e.g. 台灣大->chatGPT-> 學 台灣大->chatGPT-> (機率)學10% 歌、6% https://i.imgur.com/WxgJVLd.png https://platform.openai.com/docs/tokenizer https://i.imgur.com/OaqzMkQ.png

ChatGPT目前可以讀300頁pdt

https://i.imgur.com/LakJpvl.png

https://i.imgur.com/N38im7J.png

https://i.imgur.com/rADG2nT.png

https://i.imgur.com/p5wvO2f.png

https://i.imgur.com/LU8OEvz.png

https://i.imgur.com/sAZAVkn.png

https://i.imgur.com/jymlc5s.png

https://i.imgur.com/td5xUOQ.png

https://i.imgur.com/QUuYMwJ.png

https://i.imgur.com/d9NR3Tm.png

https://i.imgur.com/eD4pabX.png chatgpt可能跟 Instruct gpt有關

https://i.imgur.com/fnq016C.png

https://i.imgur.com/XnJi3eQ.png

https://i.imgur.com/QOxjDFZ.png

https://i.imgur.com/kG7tiSR.png

https://i.imgur.com/NFesrFO.png

https://i.imgur.com/90J8vor.png

https://i.imgur.com/R3EC4JH.png

https://i.imgur.com/E2PHJyD.png

https://i.imgur.com/8NPELGC.png gpt 3.5 人類去調整 https://i.imgur.com/Gx0UFI1.png

https://i.imgur.com/UUoGWE3.png

激發語言模型

  1. 需求講清楚
    1. e.g. 擴寫、文法
  2. 提供資訊給chatgpt
  3. 提供範例
  4. 想一想 chain of thiught
  5. 找出神奇的prompt
  6. 上傳檔案
    1. 製作投影片、外掛程式
  7. 使用其它工具
  8. 拆解任務
  9. 自主進行規劃
  10. 模型自我反省
  11. 跟真實環境互動

https://i.imgur.com/8fMEfuR.png https://i.imgur.com/HSVx09i.png

0%