W3C api 抓取

2019独角兽企业重金招聘Python工程师标准>>>

存档

W3CIDLFetcher

package canghailan.w3c;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;

import java.io.BufferedWriter;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.text.SimpleDateFormat;
import java.util.Arrays;
import java.util.Date;

/**
 * @author canghailan 2011-12-26 14:54
 */
public class W3CIDLFetcher {

    private static final int TIMEOUT = 60 * 1000; // 1min
    private static final String[] CSS_URLS = {
            "http://dev.w3.org/csswg/cssom/",
            "http://dev.w3.org/csswg/cssom-view/"
    };
    private static final String[] DOM_URLS = {
            "http://dvcs.w3.org/hg/domcore/raw-file/tip/Overview.html",
            "http://www.w3.org/TR/DOM-Level-3-Events/"
    };
    private static final String[] HTML_URLS = {
            "http://www.w3.org/TR/html5/Overview.html",
    };

    public static String[] fetchAll() throws IOException, URISyntaxException {
        Path home = Paths.get(W3CIDLFetcher.class.getProtectionDomain()
                .getCodeSource().getLocation().toURI());
        String timestamp = new SimpleDateFormat("yyyyMMddHHmmssSSS").format(new Date());
        Path cssIdl = home.resolve("css" + timestamp + ".idl");
        Path domIdl = home.resolve("dom" + timestamp + ".idl");
        Path htmlIdl = home.resolve("html" + timestamp + ".idl");

        fetch(Arrays.asList(CSS_URLS), cssIdl.toString());
        fetch(Arrays.asList(DOM_URLS), domIdl.toString());
        fetch(Arrays.asList(HTML_URLS), htmlIdl.toString());

        return new String[]{cssIdl.toString(), domIdl.toString(), htmlIdl.toString()};
    }

    public static void fetch(Iterable<String> urls, String path) throws IOException {
        for (String url : urls) {
            fetch(url, path);
        }
    }

    public static void fetch(String url, String path) throws IOException {
        System.out.println("fetch idl: " + url + " -> " + path);

        Document doc = Jsoup.connect(url).timeout(TIMEOUT).get();
        BufferedWriter writer = Files.newBufferedWriter(
                Paths.get(path), Charset.forName("UTF-8"),
                StandardOpenOption.WRITE,
                StandardOpenOption.CREATE,
                StandardOpenOption.APPEND);
        for (Element idl : doc.select("pre.idl")) {
            writer.write(idl.text());
            writer.write("\n\n");
        }
        writer.flush();
        writer.close();
    }

    public static void main(String[] args) throws IOException, URISyntaxException {
        fetchAll();
    }

}

WebIDLGrammarDownloader

package canghailan.webidl;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.BufferedWriter;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;

/**
 * @author canghailan 2011-12-26 19:39
 */
public class WebIDLGrammarDownloader {

    private static final int TIMEOUT = 60 * 1000; // 1min

    public static void download() throws IOException, URISyntaxException {
        Document doc = Jsoup.connect("http://www.w3.org/TR/WebIDL/").timeout(TIMEOUT).get();
        Elements grammars = doc.select("#appendices .grammar");

        Path home = Paths.get(WebIDLGrammarDownloader.class.getProtectionDomain()
                .getCodeSource().getLocation().toURI());
        Path webIdlGrammar = home.resolve("web-idl.g");
        BufferedWriter writer = Files.newBufferedWriter(
                webIdlGrammar, Charset.forName("UTF-8"),
                StandardOpenOption.WRITE,
                StandardOpenOption.CREATE,
                StandardOpenOption.TRUNCATE_EXISTING);

        for (Element grammar : grammars) {
            for (Element tr : grammar.select("tr")) {
                writer.write(tr.text());
                writer.write('\n');
            }
        }

        writer.flush();
        writer.close();
    }

    public static void main(String[] args) throws IOException, URISyntaxException {
        download();
    }
}

w3c-api: https://code.google.com/p/es-operating-system/

转载于:https://my.oschina.net/canghailan/blog/38431