2025-03-09T19:46:32
.300+08:00
ERROR 4464 — [ CrawlerTask-1] c
.x
.backend
.service
.PriceCrawlerService : [source1] 网络请求异常:
HTTP error fetching URL. Status=404,
URL=[
https://example
.com/price?id
=P123]package com
.xymzsfxy
.backend
.service;
import com
.xymzsfxy
.backend
.entity
.PriceHistory;
import com
.xymzsfxy
.backend
.entity
.Product;
import com
.xymzsfxy
.backend
.repository
.PriceHistoryRepository;
import com
.xymzsfxy
.backend
.repository
.ProductRepository;
import
org.jsoup.Jsoup;
import
org.junit
.jupiter
.api
.Test;
import
org.springframework
.beans
.factory
.annotation
.Autowired;
import
org.springframework
.boot
.test
.context
.SpringBootTest;
import
org.springframework
.boot
.test
.context
.TestConfiguration;
import
org.springframework
.boot
.test
.mock
.mockito
.MockBean;
import
org.springframework
.boot
.test
.mock
.mockito
.SpyBean;
import
org.springframework
.context
.annotation
.Bean;
import
org.springframework
.scheduling
.annotation
.EnableAsync;
import
org.springframework
.test
.context
.TestPropertySource;
import java
.math
.BigDecimal;
import java
.util
.Optional;
import java
.util
.concurrent
.TimeUnit;
import java
.io
.IOException;
import
org.jsoup.nodes
.Document;
import static
org.awaitility
.Awaitility
.await;
import static
org.mockito
.ArgumentMatchers
.any;
import static
org.mockito
.Mockito
.*;
// 测试类注解
@SpringBootTest
@EnableAsync
@TestPropertySource(properties
= {
“crawler
.sources
.source1
=https://example
.com/price?id
=%s”,
“crawler
.selectors
.source1
=div
.price”
})
class PriceCrawlerServiceTest {
@Autowired
private PriceCrawlerService priceCrawlerService;
@MockBean
private ProductRepository productRepository;
@MockBean
private PriceHistoryRepository priceHistoryRepository;
@SpyBean
private PriceCrawlerService spyPriceCrawlerService;
@TestConfiguration
static class TestConfig {
@Bean
public
JsoupWrapper
jsoupWrapper() {
return new
JsoupWrapper();
}
}
@Test
void shouldSavePriceWhenCrawlSuccess() throws Exception {
// 模拟测试数据
Product mockProduct
= new Product();
mockProduct
.setId(1L);
mockProduct
.setExternalId("P123");
// 配置Mock行为
when(productRepository
.findById(1L))
.thenReturn(Optional
.of(mockProduct));
doReturn(new BigDecimal("299
.00"))
.when(spyPriceCrawlerService)
.extractPrice(any(), anyString());
// 执行测试方法
priceCrawlerService
.crawlPrices(1L);
// 验证异步执行结果
await()
.atMost(5, TimeUnit
.SECONDS)
.untilAsserted(() -> {
verify(priceHistoryRepository, times(1))
.save(any(PriceHistory
.class));
verify(productRepository, times(1))
.save(argThat(p ->
p
.getLatestPrice()
.equals(new BigDecimal("299
.00"))
));
});
}
@Test
void shouldHandleMissingProduct() {
when(productRepository
.findById(999L))
.thenReturn(Optional
.empty());
// 直接调用方法，若抛出异常测试会自动失败
priceCrawlerService
.crawlPrices(999L);
verify(priceHistoryRepository, never())
.save(any());
}
}
// 辅助类用于模拟网页请求
class
JsoupWrapper {
Document fetchDocument(String
url) throws IOException {
// 模拟返回包含价格的HTML文档
String html
= “<html><div class
=‘price’>¥299
.00</div></html>”;
return
Jsoup.parse(html);
}
}如何用正确的网址验证
本文提供了一种解决JSoup爬虫在访问特定网站时遇到的HTTP 403错误的方法，即通过设置User-Agent来伪装爬虫请求，使其看起来更像常规浏览器访问。
扫一扫
1300
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
