@@ -41,7 +41,6 @@ public class NewsFeedConnector implements NewsFeedClient {
41
41
private static final String SEEKING_ALPHA_URL = "https://seekingalpha.com/market_currents.xml" ;
42
42
private static final String CNBC_FINANCE_URL = "https://search.cnbc.com/rs/search/combinedcms/view.xml?partnerId=wrss01&id=10000664" ;
43
43
private static final String SEC_EDGAR_USGAAP = "https://www.sec.gov/Archives/edgar/usgaap.rss.xml" ;
44
- private static final String SEC_EDGAR_ICON = "https://www.sec.gov/favicon.icon" ;
45
44
private static final Logger LOGGER = LoggerFactory .getLogger (NewsFeedConnector .class );
46
45
private final RestClient restClient ;
47
46
private final XmlMapper xmlMapper ;
@@ -65,11 +64,7 @@ public SyndFeed importCnbcFinanceNewsFeed() {
65
64
66
65
@ Override
67
66
public List <CompanyReportWrapper > importSecEdgarUsGaapNewsFeed () {
68
- //var result = this.loadFile(SEC_EDGAR_USGAAP, String.class);
69
- record ResultItem (String content , Boolean icon ) {}
70
- var result = List .of (SEC_EDGAR_USGAAP , SEC_EDGAR_ICON ).stream ().parallel ()
71
- .map (url -> new ResultItem (this .loadFile (url , String .class , SEC_EDGAR_ICON .equals (url )), SEC_EDGAR_ICON .equals (url )))
72
- .filter (item -> !item .icon ).findFirst ().map (item -> item .content ).orElseThrow (() -> new IllegalStateException ("No content found" ));
67
+ var result = this .loadFile (SEC_EDGAR_USGAAP , String .class );
73
68
RssDto rssDto = null ;
74
69
try {
75
70
rssDto = this .xmlMapper .readValue (result , RssDto .class );
@@ -87,24 +82,14 @@ record ResultItem(String content, Boolean icon) {}
87
82
88
83
@ Override
89
84
public byte [] loadCompanyReportZip (String url ) {
90
- return this .loadFile (url , byte [].class , false );
85
+ return this .loadFile (url , byte [].class );
91
86
}
92
87
93
- private <T > T loadFile (String url , Class <T > classType , Boolean icon ) {
88
+ private <T > T loadFile (String url , Class <T > classType ) {
94
89
var result = this .restClient .get ().uri (url )
95
90
.header ("Accept-Encoding" , "gzip, deflate" )
96
91
.header ("Host" , "www.sec.gov" )
97
- .header ("User-Agent" , "Mozilla/5.0 (X11; Linux x86_64; rv:138.0) Gecko/20100101 Firefox/138.0" )
98
- .header ("Accept" , icon ? "image/avif,image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5" : "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" )
99
- .header ("Accept-Language" , "en-US,en;q=0.5" )
100
- .header ("Accept-Encoding" , "gzip, deflate" )
101
- .header ("Connection" , "keep-alive" )
102
- .header (icon ? "Referer" : "Upgrade-Insecure-Requests" , icon ? "https://www.sec.gov/Archives/edgar/usgaap.rss.xml" : "1" )
103
- .header ("Sec-Fetch-Dest" , icon ? "image" : "document" )
104
- .header ("Sec-Fetch-Mode" , icon ? "no-cors" : "navigate" )
105
- .header ("Sec-Fetch-Site" , icon ? "same-origin" : "none" )
106
- .header (icon ? "TE" : "Sec-Fetch-User" , icon ? "trailers" : "?1" )
107
- .header ("Priority" , icon ? "u=6" : "u=0, i" )
92
+ .
header (
"User-Agent" ,
"Sven Loesekann [email protected] " )
108
93
.retrieve ().body (classType );
109
94
return result ;
110
95
}
0 commit comments