Java爬虫爬取网页内容 输出内容到Excel
更新时间：2024-11-08 分类：运营笔记 浏览量：2
目前说起爬虫，很多人都会想到 Python 语言，因为近年来关于 Python 的爬虫技术层出不穷。但爬虫不是 Python 的专利，本篇文章将会带大家用 Java 语言来爬取网页内容，并把爬取的数据导出到 Excel 文档中。
爬虫技术简述
网络爬虫（Web Crawler），按照一定的规则，自动抓取万维网信息的程序或者脚本，如今被广泛地应用在互联网搜索引擎或者其他类似网站。
爬虫在功能上分为采集、处理和储存三个部分。
爬虫基本上可以分为三大类：分布式爬虫、Java爬虫以及非Java爬虫。
在Java爬虫中又可以细分出三种，Crawler4j、WebMagic、WebCollector。
实例代码
添加依赖
<!--json-->
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.12.0</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.47</version>
</dependency>
<!--excel-->
<dependency>
<groupId>net.sourceforge.jexcelapi</groupId>
<artifactId>jxl</artifactId>
<version>2.6.12</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi</artifactId>
<version>3.17</version>
</dependency>
<dependency>
<groupId>org.apache.poi</groupId>
<artifactId>poi-ooxml</artifactId>
<version>3.17</version>
</dependency>
<!--爬虫-->
<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.13.1</version>
</dependency>创建一个Weather实体类
public class Weather {
/**
* 日期
*/
private String date;
/**
* 最高气温
*/
private String maxTemperature;
/**
* 最低气温
*/
private String minTemperature;
/**
* 白天天气
*/
private String dayTimeWeather;
/**
* 夜间天气
*/
private String nightWeather;
/**
* 风向
*/
private String windDirection;
/**
* 风力
*/
private String windPower;
public String getDate() {
return date;
}
public void setDate(String date) {
this.date = date;
}
public String getMaxTemperature() {
return maxTemperature;
}
public void setMaxTemperature(String maxTemperature) {
this.maxTemperature = maxTemperature;
}
public String getMinTemperature() {
return minTemperature;
}
public void setMinTemperature(String minTemperature) {
this.minTemperature = minTemperature;
}
public String getDayTimeWeather() {
return dayTimeWeather;
}
public void setDayTimeWeather(String dayTimeWeather) {
this.dayTimeWeather = dayTimeWeather;
}
public String getNightWeather() {
return nightWeather;
}
public void setNightWeather(String nightWeather) {
this.nightWeather = nightWeather;
}
public String getWindDirection() {
return windDirection;
}
public void setWindDirection(String windDirection) {
this.windDirection = windDirection;
}
public String getWindPower() {
return windPower;
}
public void setWindPower(String windPower) {
this.windPower = windPower;
}
@Override
public String toString() {
return "Weather{" +
"date='" + date + '\'' +
", maxTemperature='" + maxTemperature + '\'' +
", minTemperature='" + minTemperature + '\'' +
", dayTimeWeather='" + dayTimeWeather + '\'' +
", nightWeather='" + nightWeather + '\'' +
", windDirection='" + windDirection + '\'' +
", windPower='" + windPower + '\'' +
'}';
}
}创建一个WeatherTest测试类
public class WeatherTest {
public static void main(String[] args) throws FileNotFoundException, IOException {
List<Weather> list = getInfo("http://www.tianqi234.com/2020shanghai/1yue.html", 12);
for (Weather weather : list) {
System.out.println(weather.toString());
}
testHSSFWorkbook(list);
}
//可以指定网址，并且按照需求爬取前多少页的数据
public static List<Weather> getInfo(String url, int month) {
List<Weather> weatherList = new ArrayList<Weather>();
for (int i = 1; i < month + 1; i++) {
try {
System.out.println("url:" + url);
Document doc = Jsoup.connect(url).get();
Elements table = doc.select(".graybox_cnt");
/* Elements tbody = table.select("tbody");*/
Elements trList = table.select("tr");
//每次移除的时候,你的列表长度就会发生新的变化,所以要结合实际进行移除
trList.remove(0);
if (i > 1) {
trList.remove(0);
trList.remove(10);
trList.remove(10);
trList.remove(20);
trList.remove(20);
trList.remove(20);
} else {
trList.remove(11);
trList.remove(11);
trList.remove(21);
trList.remove(21);
trList.remove(21);
}
for (Element tr : trList) {
Elements tdList = tr.select("td");
Elements aList = tdList.select("a");//查询a标签
Weather weather = new Weather();
if (aList != null && aList.size() > 0) {
weather.setDate(aList.get(0).html().toString());
} else {
weather.setDate(tdList.get(0).html().toString());
}
weather.setMaxTemperature(tdList.get(1).html().toString());
weather.setMinTemperature(tdList.get(2).html().toString());
weather.setDayTimeWeather(tdList.get(3).html().toString());
weather.setNightWeather(tdList.get(4).html().toString());
weather.setWindDirection(tdList.get(5).html().toString());
weather.setWindPower(tdList.get(6).html().toString());
weatherList.add(weather);
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
url = "http://www.tianqi234.com/2020shanghai/" + (i + 1) + "yue.html";
}
return weatherList;
}
public static void testHSSFWorkbook(List<Weather> list) throws IOException {
HSSFWorkbook workbook = new HSSFWorkbook();//创建excel文件(workbook)
HSSFSheet sheet = workbook.createSheet("2020年上海天气统计");
HSSFRow row = sheet.createRow(0);//创建行 从0开始
HSSFCellStyle style = workbook.createCellStyle();//设置单元格样式
style.setAlignment(HorizontalAlignment.CENTER);//水平居中
style.setVerticalAlignment(VerticalAlignment.CENTER);//垂直居中
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
sheet.setDefaultColumnWidth(30);
row.setHeightInPoints(25);
Map<String, String> map = (Map<String, String>) getMap(list.get(0));
//设置表头
int c = 0;
for (String key : map.keySet()) {
HSSFCell cell = row.createCell(c);//创建行的单元格，从0开始
cell.setCellValue(map.get(key));//设置单元格内容
cell.setCellStyle(style);
c++;
}
Map<Integer, Weather> weatherMap = new HashMap<>();
//除去表头
for (int i = 1; i < list.size(); i++) {
weatherMap.put(i, list.get(i));
}
for (int i = 1; i <= weatherMap.size(); i++) {
HSSFRow rowInfo = sheet.createRow(i);
rowInfo.setHeightInPoints(30);
Map<String, String> map1 = (Map<String, String>) getMap(list.get(i));
int j = 0;
for (String key : map1.keySet()) {
HSSFCell cellInfo = rowInfo.createCell(j);
cellInfo.setCellValue(map1.get(key));
cellInfo.setCellStyle(style);
j++;
}
}
FileOutputStream out = new FileOutputStream("D:\\weather1.xlsx");
workbook.write(out);
out.close();
}
/**
* json转map
*
* @param object
* @return
*/
public static Map<?, ?> getMap(Object object) {
if (object == null) {
throw new RuntimeException("对象为空,转json失败");
}
Map<String, Object> map = new HashMap<>();
try {
map = (Map) JSON.parse(JSON.toJSONString(object));
} catch (Exception e) {
System.out.println("对象转map转换失败");
}
return map;
}
}最后结果展示
总结
本篇关于使用Java爬虫爬取网页内容，并导出到Excel文档的内容就介绍到此结束了。想要了解更多和Java有关的知识，请关注W3Cschool以前的文章或者继续浏览接下来的内容，也希望大家能够多多支持我们。