使用jetty和mongodb实现简易网盘接口

时间：2023-01-21 22:08:48

依赖库：

1，jetty（提供http方式接口）

2，mongodb的java驱动（访问mongodb存取文件）

3，thumbnailator包，进行缩略图生成

4，commons-fileupload包及commons-io包用于处理文件上传

架构图：

使用jetty和mongodb实现简易网盘接口

入口代码如下：

package com.ciaos.vfs;

import org.eclipse.jetty.server.Connector;

import org.eclipse.jetty.server.Server;

import org.eclipse.jetty.server.nio.SelectChannelConnector;

import org.eclipse.jetty.servlet.ServletContextHandler;

import org.eclipse.jetty.servlet.ServletHolder;

import org.eclipse.jetty.util.thread.QueuedThreadPool;

public class VfsServer {

    public static void main(String[] args) throws Exception {

        SelectChannelConnector connector = new SelectChannelConnector();

        connector.setHost("127.0.0.1");

        connector.setPort(8080);

        connector.setThreadPool(new QueuedThreadPool(200));

        Server server = new Server();

        server.setConnectors(new Connector[]{connector}); 

        ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);

        context.setContextPath("/");  

        server.setHandler(context);  

        context.addServlet(new ServletHolder(new VfsFileServlet()), "/file");

        server.start();

        server.join();

    }

}

业务代码如下：

package com.ciaos.vfs;

import java.awt.image.BufferedImage;

import java.io.File;

import java.io.IOException;

import java.io.InputStream;

import java.net.UnknownHostException;

import java.util.List;

import javax.imageio.ImageIO;

import javax.servlet.ServletException;

import javax.servlet.ServletOutputStream;

import javax.servlet.http.HttpServlet;

import javax.servlet.http.HttpServletRequest;

import javax.servlet.http.HttpServletResponse;  

import net.coobird.thumbnailator.Thumbnails;

import org.apache.commons.fileupload.FileItem;

import org.apache.commons.fileupload.disk.DiskFileItemFactory;

import org.apache.commons.fileupload.servlet.ServletFileUpload;

import org.bson.types.ObjectId;

import com.mongodb.DB;

import com.mongodb.Mongo;

import com.mongodb.gridfs.GridFS;

import com.mongodb.gridfs.GridFSDBFile;

import com.mongodb.gridfs.GridFSInputFile;

public class VfsFileServlet extends HttpServlet {

    /**

     *

     */

    private static final long serialVersionUID = 5586455171943232770L;

    private String tempPath = null;

    private Mongo mongo = null;

    final private String MongoDbIp = "127.0.0.1";

    final private Integer MongoDbPort = 27017;

    @Override

    public void init() throws ServletException {

        // TODO Auto-generated method stub

        tempPath = "/tmp/";

        try {

            mongo=new Mongo(MongoDbIp, MongoDbPort);

        } catch (UnknownHostException e) {

            // TODO Auto-generated catch block

            e.printStackTrace();

            System.exit(-1);

        }

        super.init();

    }

    @Override

    protected void doDelete(HttpServletRequest req, HttpServletResponse resp)

            throws ServletException, IOException {

        // TODO Auto-generated method stub

        String fileId = req.getParameter("id");

        String MongoDbName = req.getParameter("type");

        if(fileId == null || MongoDbName == null){

            resp.sendError(400,"bad request");

            return;

        }

        DB db=mongo.getDB(MongoDbName);

        GridFS gridFS=new GridFS(db);

        try{

            gridFS.remove(new ObjectId(fileId));

        }

        catch(IllegalArgumentException ex){

            resp.sendError(404,"file not found");

            return;

        }

    }

    @Override

    protected void doGet(HttpServletRequest req, HttpServletResponse resp)

            throws ServletException, IOException {

        // TODO Auto-generated method stub

        String fileId = req.getParameter("id");

        String MongoDbName = req.getParameter("type");

        if(fileId == null || MongoDbName == null){

            resp.sendError(400,"bad request");

            return;

        }

        DB db=mongo.getDB(MongoDbName);

        GridFS gridFS=new GridFS(db);

        GridFSDBFile mongofile = null;

        try{

            mongofile = gridFS.find(new ObjectId(fileId));

        }

        catch(IllegalArgumentException ex){

            resp.sendError(404,"file not found");

            return;

        }

        if(mongofile!=null){

            resp.setHeader("Content-type", mongofile.getContentType());

            resp.setHeader("Content-Disposition", "attachment; filename= "+mongofile.getFilename());

            mongofile.put("viewtimes", (Integer)mongofile.get("viewtimes")+1);

            ServletOutputStream out = resp.getOutputStream();

            String dtype = req.getParameter("act");

            if(dtype!=null && dtype.equals("preview")){

                Integer width = 0, height = 0;

                try{

                    width = Integer.parseInt(req.getParameter("w"));

                    height = Integer.parseInt(req.getParameter("h"));

                    BufferedImage image = ImageIO.read(mongofile.getInputStream());

                    width = Math.min(width,image.getWidth());

                    height = Math.min(height,image.getHeight());

                    Thumbnails.of(mongofile.getInputStream())

                            .size(width, height)

                            .toOutputStream(out);

                }

                catch(NumberFormatException ex){

                    resp.sendError(400,"bad request");

                    return;

                }

            }else{

                InputStream input = mongofile.getInputStream();

                byte[] buffer = new byte[64*1024];

                for(;;)

                {

                    int count = input.read(buffer);

                    if (count < 0)

                        break;

                    out.write(buffer,0,count);

                }

            }

            out.flush();

            out.close();

            mongofile.save();

        }else{

            resp.sendError(500,"internal server error");

        }

    }

    @Override

    protected void doPost(HttpServletRequest req, HttpServletResponse resp)

            throws ServletException, IOException {

        // TODO Auto-generated method stub

        doPut(req, resp);

    }

    @Override

    protected void doPut(HttpServletRequest req, HttpServletResponse resp)

            throws ServletException, IOException {

        // TODO Auto-generated method stub

        req.setCharacterEncoding("utf-8");

        DiskFileItemFactory factory = new DiskFileItemFactory();

        factory.setRepository(new File(tempPath));

        factory.setSizeThreshold(8 * 1024 * 1024);

        ServletFileUpload upload = new ServletFileUpload(factory);

        try {

            List<FileItem> list = (List<FileItem>) upload.parseRequest(req);

            for (FileItem item : list) {

                if (item.isFormField()) {

                } else {

                    String value = item.getName();

                    int start = value.lastIndexOf("\\");

                    String filename = value.substring(start + 1);    

                    String MongoDbName = req.getParameter("type");

                    if(MongoDbName == null){

                        resp.sendError(400,"bad request");

                        return;

                    }

                    DB db=mongo.getDB(MongoDbName);

                    GridFS gridFS=new GridFS(db);

                    GridFSInputFile  mongofile=gridFS.createFile(item.getInputStream());

                    mongofile.put("viewtimes",0);

                    mongofile.setContentType(item.getContentType());

                    mongofile.put("filename", filename);

                    mongofile.save();

                    resp.getWriter().println(mongofile);

                    resp.getWriter().flush();

                }

            }

        } catch (Exception e) {

            e.printStackTrace();

            resp.sendError(500,"internal server error");

        }

    }

}

运行方法一（eclipse很简单不用介绍，linux命令行运行方法，大于8M文件通过/tmp/目录缓存）：

javac -cp .:../libs/* com/speakbang/vfs/*.java -Xlint:deprecation

java -cp .:../libs/* com/speakbang/vfs/VfsServer

运行方法二（只需要VfsFileServlet.java，集成到jetty配置文件）

root:~/jetty/webapps # ls -R *

ROOT:

    WEB-INF

        classes

            com/ciaos/vfs/VfsFileServlet.class

        lib

            commons-fileupload-1.3.1.jar  commons-io-2.4.jar  mongo-java-driver-2.11.4.jar  thumbnailator-0.4.7.jar

        src

            com/ciaos/vfs/VfsFileServlet.java

        web.xml

web.xml

<?xml version="1.0" encoding="ISO-8859-1"?>

<web-app xmlns="http://java.sun.com/xml/ns/javaee"

        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"

        xsi:schemaLocation="http://java.sun.com/xml/ns/javaee http://java.sun.com/xml/ns/javaee/web-app_2_5.xsd"

        version="2.5">

        <servlet>

        <servlet-name>VfsServlet</servlet-name>

        <servlet-class>com.ciaos.vfs.VfsFileServlet</servlet-class>

        <load-on-startup>0</load-on-startup>

        </servlet>

        <servlet-mapping>

        <servlet-name>VfsServlet</servlet-name>

        <url-pattern>/file</url-pattern>

        </servlet-mapping>

</web-app>
cd webapps/ROOT/WEB-INF/src/

javac -cp .:../lib/*:../../../../lib/* com/ciaos/vfs/VfsFileServlet.java -Xlint:deprecation

mv com/ciaos/vfs/VfsFileServlet.class ../classes/com/ciaos/vfs/

cd ../../../../

java -jar start.jar -Xms128m -Xmx256m
(bin/jetty.sh start|restart|stop)

运行方法三（将class文件打成jar包，放到lib目录下，可以不用放置src和classes目录及源码了）

cd classes/

jar cvf vfs.jar com

cd ..

mv classes/vfs.jar lib/

rm -rf classes src

cd ../../

java -jar start.jar

注意文件不能直接操作db.fs.files.remove()删除，fs.chunks这个collection里面才是包含文件内容的，正确的删除方法使用如下脚本或者调用DELETE接口，删除完毕后用db.repairDatabase()释放磁盘空间：

var year=2014

var month=3

var day=15

use test

var items = db.fs.files.find({uploadDate:{$lte:new Date(year+"/"+month+"/"+day)}})

items.forEach(function(item) {

        db.fs.chunks.remove({files_id: item._id})

        db.fs.files.remove({_id:item._id})

        print(item._id + " deleted")

})

//运行方法： bin/mongo < tool/delete.js

//root:~/mongodb$ bin/mongo < tool/delete.js

//MongoDB shell version: 2.4.9

//connecting to: test

//switched to db test

//532259170cf2c0bcb68e1e72 deleted

//532259190cf2c0bcb68e1e76 deleted

//5322591a0cf2c0bcb68e1e7a deleted

//bye

使用方法：

上传：curl -F "action=upload" -F "Filedata=@a.png;type=image/png" -v "http://127.0.0.1:8080/file?type=test"

{ "_id" : { "$oid" : "531c5fe6744e22f4c22eeef5"} , "chunkSize" : 262144 , "length" : 1033834 , "md5" : "04bc7c0988ecc04f93cfa96f239dda99" , "filename" : "a.png" , "contentType" :  null  , "viewtimes" : 0 , "uploadDate" : { "$date" : "2014-03-09T12:34:46.347Z"} , "aliases" :  null }

下载：curl -o /dev/null -v http://127.0.0.1:8080/file?type=test&id=531c5fe6744e22f4c22eeef5&act=preview&w=200&h=200

删除:curl -X DELETE -v "http://127.0.0.1:8080/file?type=test&id=531c5fe6744e22f4c22eeef5"

Java代码上传方法（支持多文件上传）

package com.test;

import java.io.BufferedReader;

import java.io.File;

import java.io.FileInputStream;

import java.io.IOException;

import java.io.InputStreamReader;

import java.io.OutputStream;

import java.io.OutputStreamWriter;

import java.io.PrintWriter;

import java.net.HttpURLConnection;

import java.net.URL;

import java.net.URLConnection;

import java.util.ArrayList;

import java.util.List;

/**

 * This utility class provides an abstraction layer for sending multipart HTTP

 * POST requests to a web server.

 * @author www.codejava.net

 *

 */

public class MultipartUtility {

    private final String boundary;

    private static final String LINE_FEED = "\r\n";

    private HttpURLConnection httpConn;

    private String charset;

    private OutputStream outputStream;

    private PrintWriter writer;

    /**

     * This constructor initializes a new HTTP POST request with content type

     * is set to multipart/form-data

     * @param requestURL

     * @param charset

     * @throws IOException

     */

    public MultipartUtility(String requestURL, String charset)

            throws IOException {

        this.charset = charset;

        // creates a unique boundary based on time stamp

        boundary = "===" + System.currentTimeMillis() + "===";

        URL url = new URL(requestURL);

        httpConn = (HttpURLConnection) url.openConnection();

        httpConn.setUseCaches(false);

        httpConn.setDoOutput(true); // indicates POST method

        httpConn.setDoInput(true);

        httpConn.setRequestProperty("Content-Type",

                "multipart/form-data; boundary=" + boundary);

        outputStream = httpConn.getOutputStream();

        writer = new PrintWriter(new OutputStreamWriter(outputStream, charset),

                true);

    }

    /**

     * Adds a form field to the request

     * @param name field name

     * @param value field value

     */

    public void addFormField(String name, String value) {

        writer.append("--" + boundary).append(LINE_FEED);

        writer.append("Content-Disposition: form-data; name=\"" + name + "\"")

                .append(LINE_FEED);

        writer.append("Content-Type: text/plain; charset=" + charset).append(

                LINE_FEED);

        writer.append(LINE_FEED);

        writer.append(value).append(LINE_FEED);

        writer.flush();

    }

    /**

     * Adds a upload file section to the request

     * @param fieldName name attribute in <input type="file" name="..." />

     * @param uploadFile a File to be uploaded

     * @throws IOException

     */

    public void addFilePart(String fieldName, File uploadFile)

            throws IOException {

        String fileName = uploadFile.getName();

        writer.append("--" + boundary).append(LINE_FEED);

        writer.append(

                "Content-Disposition: form-data; name=\"" + fieldName

                        + "\"; filename=\"" + fileName + "\"")

                .append(LINE_FEED);

        writer.append(

                "Content-Type: "

                        + URLConnection.guessContentTypeFromName(fileName))

                .append(LINE_FEED);

        writer.append("Content-Transfer-Encoding: binary").append(LINE_FEED);

        writer.append(LINE_FEED);

        writer.flush();

        FileInputStream inputStream = new FileInputStream(uploadFile);

        byte[] buffer = new byte[4096];

        int bytesRead = -1;

        while ((bytesRead = inputStream.read(buffer)) != -1) {

            outputStream.write(buffer, 0, bytesRead);

        }

        outputStream.flush();

        inputStream.close();

        writer.append(LINE_FEED);

        writer.flush();

    }

    /**

     * Adds a header field to the request.

     * @param name - name of the header field

     * @param value - value of the header field

     */

    public void addHeaderField(String name, String value) {

        writer.append(name + ": " + value).append(LINE_FEED);

        writer.flush();

    }

    /**

     * Completes the request and receives response from the server.

     * @return a list of Strings as response in case the server returned

     * status OK, otherwise an exception is thrown.

     * @throws IOException

     */

    public List<String> finish() throws IOException {

        List<String> response = new ArrayList<String>();

        writer.append(LINE_FEED).flush();

        writer.append("--" + boundary + "--").append(LINE_FEED);

        writer.close();

        // checks server's status code first

        int status = httpConn.getResponseCode();

        if (status == HttpURLConnection.HTTP_OK) {

            BufferedReader reader = new BufferedReader(new InputStreamReader(

                    httpConn.getInputStream()));

            String line = null;

            while ((line = reader.readLine()) != null) {

                response.add(line);

            }

            reader.close();

            httpConn.disconnect();

        } else {

            throw new IOException("Server returned non-OK status: " + status);

        }

        return response;

    }

    public static void main(String[] args) {

        String charset = "UTF-8";

        File uploadFile1 = new File("c:/1.jpg");

        File uploadFile2 = new File("c:/2.jpg");

        String requestURL = "http://localhost:8080/file?type=test";

        try {

            MultipartUtility multipart = new MultipartUtility(requestURL, charset);

/*

            multipart.addHeaderField("User-Agent", "CodeJava");

            multipart.addHeaderField("Test-Header", "Header-Value");

            multipart.addFormField("description", "Cool Pictures");

            multipart.addFormField("keywords", "Java,upload,Spring");

*/

            multipart.addFilePart("fileUpload", uploadFile1);

            multipart.addFilePart("fileUpload", uploadFile2);

            List<String> response = multipart.finish();

            for (String line : response) {

                System.out.println(line);

            }

            //output

            //533e27560cf2f5c3e3d77748

            //533e27560cf2f5c3e3d7774a

        } catch (IOException ex) {

            System.err.println(ex);

        }

    }

}

相关文章