230421/Scrapper
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

noticias adicao

Browse Source
This commit is contained in:
230421
2026-04-27 15:00:45 +01:00
parent 18658058de
commit 02a235c114
5 changed files with 409 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.idea/.name generated Normal file
View File

@@ -0,0 +1 @@
testeScraper

108
src/main/java/org/example/NewsScraper.java Normal file
View File

@@ -0,0 +1,108 @@
package org.example;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.firebase.FirebaseApp;
import com.google.firebase.FirebaseOptions;
import com.google.firebase.database.DatabaseReference;
import com.google.firebase.database.FirebaseDatabase;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.example.models.News;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class NewsScraper {
public static void main(String[] args) {
String caminhoChave = "service-account.json";
String urlDatabase = "https://vdcscore-default-rtdb.firebaseio.com/";
try {
System.out.println("--- A CONECTAR AO FIREBASE ---");
if (FirebaseApp.getApps().isEmpty()) {
FileInputStream serviceAccount = new FileInputStream(caminhoChave);
FirebaseOptions options = FirebaseOptions.builder()
.setCredentials(GoogleCredentials.fromStream(serviceAccount))
.setDatabaseUrl(urlDatabase)
.build();
FirebaseApp.initializeApp(options);
}
} catch (Exception e) {
System.err.println("ERRO FIREBASE: Verifica o service-account.json!");
e.printStackTrace();
return;
}
Gson gson = new Gson();
System.out.println("\n--- A EXTRAIR NOTÍCIAS ---");
try {
// Vamos extrair as primeiras páginas de notícias (ex: 1 a 3)
// Para o resumo na app, talvez apenas a primeira página (mais recentes) baste.
String urlNews = "https://api.afavcd.pt/news/1";
String jsonNews = Jsoup.connect(urlNews)
.ignoreContentType(true)
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
.header("Referer", "https://www.afavcd.pt/")
.header("Origin", "https://www.afavcd.pt")
.method(Connection.Method.GET)
.execute()
.body();
Map<String, Object> responseMap = gson.fromJson(jsonNews, new TypeToken<Map<String, Object>>() {}.getType());
List<Map<String, Object>> rows = (List<Map<String, Object>>) responseMap.get("rows");
List<News> newsList = new ArrayList<>();
if (rows != null) {
for (Map<String, Object> row : rows) {
News n = new News();
if (row.get("newsID") != null) n.setNewsID((int) Double.parseDouble(row.get("newsID").toString()));
if (row.get("title") != null) n.setTitle(row.get("title").toString());
if (row.get("body") != null) n.setBody(row.get("body").toString());
if (row.get("insertDate") != null) n.setInsertDate(row.get("insertDate").toString());
String photo = null;
if (row.get("photoURl") != null) {
photo = row.get("photoURl").toString();
} else if (row.get("photoURL") != null) {
photo = row.get("photoURL").toString();
}
if (photo != null && !photo.isEmpty() && !photo.startsWith("http")) {
photo = "https://api.afavcd.pt" + photo;
}
n.setPhotoURL(photo);
newsList.add(n);
}
}
System.out.println("Encontradas " + newsList.size() + " notícias.");
// Enviar para Firebase
DatabaseReference refNoticias = FirebaseDatabase.getInstance().getReference("noticias");
refNoticias.setValueAsync(newsList);
System.out.println("-> Notícias enviadas para o Firebase.");
} catch (Exception e) {
System.err.println("Erro ao obter notícias.");
e.printStackTrace();
}
try {
System.out.println("\nAguardando confirmação do servidor (5s)...");
Thread.sleep(5000);
System.out.println("Processo concluído!");
System.exit(0);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}

178
src/main/java/org/example/TopScorersScraper.java Normal file
View File

@@ -0,0 +1,178 @@
package org.example;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.firebase.FirebaseApp;
import com.google.firebase.FirebaseOptions;
import com.google.firebase.database.DatabaseReference;
import com.google.firebase.database.FirebaseDatabase;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.example.models.Club;
import org.example.models.TopScorer;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import java.io.FileInputStream;
import java.util.*;
public class TopScorersScraper {
public static void main(String[] args) {
String caminhoChave = "service-account.json";
String urlDatabase = "https://vdcscore-default-rtdb.firebaseio.com/";
try {
System.out.println("--- A CONECTAR AO FIREBASE ---");
if (FirebaseApp.getApps().isEmpty()) {
FileInputStream serviceAccount = new FileInputStream(caminhoChave);
FirebaseOptions options = FirebaseOptions.builder()
.setCredentials(GoogleCredentials.fromStream(serviceAccount))
.setDatabaseUrl(urlDatabase)
.build();
FirebaseApp.initializeApp(options);
}
} catch (Exception e) {
System.err.println("ERRO FIREBASE: Verifica o service-account.json!");
e.printStackTrace();
return;
}
Gson gson = new Gson();
Map<Integer, Club> clubesMap = new HashMap<>();
// 1. Obter todos os Clubes para ter logos e nomes reais
System.out.println("\n--- A EXTRAIR DADOS DOS CLUBES ---");
try {
String urlTodosClubes = "https://api.afavcd.pt/teams";
String jsonTodosClubes = Jsoup.connect(urlTodosClubes)
.ignoreContentType(true)
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
.header("Referer", "https://www.afavcd.pt/")
.header("Origin", "https://www.afavcd.pt")
.maxBodySize(0)
.timeout(30000)
.method(Connection.Method.GET)
.execute()
.body();
List<Map<String, Object>> dadosClubes = gson.fromJson(jsonTodosClubes,
new TypeToken<List<Map<String, Object>>>() {}.getType());
for (Map<String, Object> c : dadosClubes) {
if (c.get("teamID") != null) {
int clubId = (int) Double.parseDouble(c.get("teamID").toString());
Club club = new Club();
club.setId(clubId);
if (c.get("name") != null) club.setName(c.get("name").toString());
if (c.get("logoURL") != null) {
String logoUrl = c.get("logoURL").toString();
if (logoUrl != null && !logoUrl.startsWith("http") && !logoUrl.isEmpty()) {
logoUrl = "https://api.afavcd.pt" + logoUrl;
}
club.setImageUrl(logoUrl);
}
clubesMap.put(clubId, club);
}
}
System.out.println("Encontrados " + clubesMap.size() + " clubes.");
} catch (Exception e) {
System.err.println("Erro ao obter lista de clubes.");
e.printStackTrace();
return;
}
int[] modalidades = { 1, 3 }; // 1: Seniores, 3: Juniores
String[] nomesEscaloes = { "seniores", "juniores" };
for (int i = 0; i < modalidades.length; i++) {
int modalidade = modalidades[i];
String escalao = nomesEscaloes[i];
System.out.println("\n--- A EXTRAIR MARCADORES PARA: " + escalao.toUpperCase() + " ---");
String urlDiscipline = "https://api.afavcd.pt/teams/modality/" + modalidade + "/season/33/discipline";
try {
String jsonDiscipline = Jsoup.connect(urlDiscipline)
.ignoreContentType(true)
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
.header("Referer", "https://www.afavcd.pt/")
.header("Origin", "https://www.afavcd.pt")
.maxBodySize(0)
.timeout(60000)
.method(Connection.Method.GET)
.execute()
.body();
Map<String, Object> root = gson.fromJson(jsonDiscipline, new TypeToken<Map<String, Object>>() {}.getType());
Map<String, List<Map<String, Object>>> teams = (Map<String, List<Map<String, Object>>>) root.get("teams");
List<TopScorer> scorersList = new ArrayList<>();
if (teams != null) {
for (List<Map<String, Object>> players : teams.values()) {
for (Map<String, Object> player : players) {
String playerName = (String) player.get("fullName");
String playerPhoto = (String) player.get("photoURl");
int clubId = (int) Double.parseDouble(player.get("teamID").toString());
Club club = clubesMap.get(clubId);
String clubName = club != null ? club.getName() : (String) player.get("name");
String clubLogo = club != null ? club.getImageUrl() : "";
List<Map<String, Object>> jorneys = (List<Map<String, Object>>) player.get("jorneys");
int totalGoals = 0;
if (jorneys != null) {
for (Map<String, Object> j : jorneys) {
Object goalsObj = j.get("totalGoals");
if (goalsObj != null && !goalsObj.toString().isEmpty()) {
try {
totalGoals += (int) Double.parseDouble(goalsObj.toString());
} catch (Exception ignored) {}
}
}
}
if (totalGoals > 0) {
TopScorer ts = new TopScorer();
ts.setPlayerName(playerName);
ts.setPlayerPhoto(playerPhoto != null && !playerPhoto.startsWith("http") ? "https://api.afavcd.pt" + playerPhoto : playerPhoto);
ts.setClubName(clubName);
ts.setClubLogo(clubLogo);
ts.setGoals(totalGoals);
scorersList.add(ts);
}
}
}
}
// Ordenar por golos
Collections.sort(scorersList);
// Definir posições e limitar aos top 20 (opcional, mas comum)
for (int j = 0; j < scorersList.size(); j++) {
scorersList.get(j).setPosition(j + 1);
}
// Enviar para Firebase
DatabaseReference refMarcadores = FirebaseDatabase.getInstance()
.getReference("marcadores").child(escalao);
refMarcadores.setValueAsync(scorersList);
System.out.println("-> " + scorersList.size() + " marcadores de " + escalao + " enviados para o Firebase.");
} catch (Exception e) {
System.err.println("Erro ao processar marcadores de " + escalao);
e.printStackTrace();
}
}
try {
System.out.println("\nAguardando confirmação do servidor (5s)...");
Thread.sleep(5000);
System.out.println("Processo concluído!");
System.exit(0);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}

52
src/main/java/org/example/models/News.java Normal file
View File

@@ -0,0 +1,52 @@
package org.example.models;
public class News {
private int newsID;
private String title;
private String body;
private String insertDate;
private String photoURL;
public News() {
}
public int getNewsID() {
return newsID;
}
public void setNewsID(int newsID) {
this.newsID = newsID;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getBody() {
return body;
}
public void setBody(String body) {
this.body = body;
}
public String getInsertDate() {
return insertDate;
}
public void setInsertDate(String insertDate) {
this.insertDate = insertDate;
}
public String getPhotoURL() {
return photoURL;
}
public void setPhotoURL(String photoURL) {
this.photoURL = photoURL;
}
}

70
src/main/java/org/example/models/TopScorer.java Normal file
View File

@@ -0,0 +1,70 @@
package org.example.models;
import java.io.Serializable;
public class TopScorer implements Serializable, Comparable<TopScorer> {
private String playerName;
private String playerPhoto;
private String clubName;
private String clubLogo;
private int goals;
private int position;
public TopScorer() {
}
public String getPlayerName() {
return playerName;
}
public void setPlayerName(String playerName) {
this.playerName = playerName;
}
public String getPlayerPhoto() {
return playerPhoto;
}
public void setPlayerPhoto(String playerPhoto) {
this.playerPhoto = playerPhoto;
}
public String getClubName() {
return clubName;
}
public void setClubName(String clubName) {
this.clubName = clubName;
}
public String getClubLogo() {
return clubLogo;
}
public void setClubLogo(String clubLogo) {
this.clubLogo = clubLogo;
}
public int getGoals() {
return goals;
}
public void setGoals(int goals) {
this.goals = goals;
}
public int getPosition() {
return position;
}
public void setPosition(int position) {
this.position = position;
}
@Override
public int compareTo(TopScorer other) {
// Ordenar por golos decrescente
return Integer.compare(other.getGoals(), this.getGoals());
}
}