5 Commits

Author SHA1 Message Date
da558c5a90 adicao do campo taça no fireabse 2026-05-12 17:10:21 +01:00
cbdaa8a5e5 adicao do campo "played" nas jornadas firebase 2026-04-30 10:23:45 +01:00
02a235c114 noticias adicao 2026-04-27 15:00:45 +01:00
18658058de atualizacoes 2026-04-27 15:00:27 +01:00
a7bd44ba18 adicao de jornadas 2026-04-24 16:30:48 +01:00
9 changed files with 660 additions and 6 deletions

1
.idea/.name generated Normal file
View File

@@ -0,0 +1 @@
testeScraper

2
.idea/vcs.xml generated
View File

@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
<mapping directory="" vcs="Git" />
</component>
</project>

View File

@@ -0,0 +1,153 @@
package org.example;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.firebase.FirebaseApp;
import com.google.firebase.FirebaseOptions;
import com.google.firebase.database.DatabaseReference;
import com.google.firebase.database.FirebaseDatabase;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import java.io.FileInputStream;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
public class CupScraper {
public static void main(String[] args) {
String caminhoChave = "service-account.json";
String urlDatabase = "https://vdcscore-default-rtdb.firebaseio.com/";
try {
System.out.println("--- A CONECTAR AO FIREBASE ---");
if (FirebaseApp.getApps().isEmpty()) {
FileInputStream serviceAccount = new FileInputStream(caminhoChave);
FirebaseOptions options = FirebaseOptions.builder()
.setCredentials(GoogleCredentials.fromStream(serviceAccount))
.setDatabaseUrl(urlDatabase)
.build();
FirebaseApp.initializeApp(options);
}
} catch (Exception e) {
System.err.println("ERRO FIREBASE!");
e.printStackTrace();
return;
}
Gson gson = new Gson();
String[] urls = {
"https://api.afavcd.pt/cup/seasons/33/modality/1",
"https://api.afavcd.pt/cup/seasons/33/modality/3"
};
String[] escaloes = { "seniores", "juniores" };
DatabaseReference refTaca = FirebaseDatabase.getInstance().getReference("taça");
for (int i = 0; i < urls.length; i++) {
String url = urls[i];
String escalao = escaloes[i];
System.out.println("\n=== A PROCESSAR TAÇA PARA: " + escalao.toUpperCase() + " ===");
try {
String json = Jsoup.connect(url)
.ignoreContentType(true)
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
.header("Referer", "https://www.afavcd.pt/")
.header("Origin", "https://www.afavcd.pt")
.method(Connection.Method.GET)
.execute()
.body();
Map<String, Object> root = gson.fromJson(json, new TypeToken<Map<String, Object>>() {}.getType());
if (!root.containsKey("games")) {
System.out.println("Sem jogos para " + escalao);
continue;
}
@SuppressWarnings("unchecked")
Map<String, List<Map<String, Object>>> phases = (Map<String, List<Map<String, Object>>>) root.get("games");
for (Map.Entry<String, List<Map<String, Object>>> entry : phases.entrySet()) {
String phaseKeyRaw = entry.getKey();
// Firebase friendly phase name
String phaseKey = phaseKeyRaw.replaceAll("[\\.#\\$\\[\\]]", "_");
for (Map<String, Object> game : entry.getValue()) {
try {
Map<String, Object> matchMap = new HashMap<>();
matchMap.put("home_nome", game.get("homeName"));
matchMap.put("away_nome", game.get("awayName"));
matchMap.put("home_logo", game.get("homeLogo"));
matchMap.put("away_logo", game.get("awayLogo"));
matchMap.put("cupJorney", game.get("cupJorney"));
String hGoals = String.valueOf(game.get("homeGoals"));
String aGoals = String.valueOf(game.get("awayGoals"));
boolean played = false;
if (hGoals != null && !hGoals.equals("null") && !hGoals.trim().isEmpty()) {
matchMap.put("home_golos", hGoals); // Some cup scores might be like "1p"
played = true;
}
if (aGoals != null && !aGoals.equals("null") && !aGoals.trim().isEmpty()) {
matchMap.put("away_golos", aGoals);
played = true;
}
matchMap.put("played", played);
Object dateObj = game.get("date");
if (dateObj != null && !String.valueOf(dateObj).equals("null")) {
String dateStr = String.valueOf(dateObj);
if (dateStr.contains("T")) {
String[] parts = dateStr.split("T");
matchMap.put("data", parts[0]);
if (parts[1].length() >= 5) {
matchMap.put("hora", parts[1].substring(0, 5));
}
} else {
matchMap.put("data", dateStr);
}
}
Object fieldObj = game.get("field");
if (fieldObj != null && !String.valueOf(fieldObj).equals("null")) {
matchMap.put("campo", String.valueOf(fieldObj));
}
Object reportUrlObj = game.get("reportURL");
if (reportUrlObj != null && !String.valueOf(reportUrlObj).equals("null") && !String.valueOf(reportUrlObj).trim().isEmpty()) {
matchMap.put("matchReportUrl", "https://api.afavcd.pt" + String.valueOf(reportUrlObj));
}
String cupID = String.valueOf(game.get("cupID"));
if (cupID.contains(".")) {
cupID = cupID.substring(0, cupID.indexOf("."));
}
refTaca.child(escalao).child(phaseKey).child(cupID).setValueAsync(matchMap);
} catch (Exception ex) {
System.err.println("Erro ao processar jogo: " + ex.getMessage());
}
}
}
System.out.println("-> Dados de " + escalao + " agendados para envio.");
} catch (Exception e) {
System.err.println("Erro a obter taça para " + escalao);
e.printStackTrace();
}
}
try {
System.out.println("\nAguardando confirmação do servidor (10s)...");
Thread.sleep(10000);
System.out.println("Processo de Taça concluído!");
System.exit(0);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}

View File

@@ -66,8 +66,13 @@ public class Main {
club.setId(clubId);
if (c.get("name") != null)
club.setName(c.get("name").toString());
if (c.get("logoURL") != null)
club.setImageUrl(c.get("logoURL").toString());
if (c.get("logoURL") != null) {
String logoUrl = c.get("logoURL").toString();
if (logoUrl != null && !logoUrl.startsWith("http") && !logoUrl.isEmpty()) {
logoUrl = "https://api.afavcd.pt" + logoUrl;
}
club.setImageUrl(logoUrl);
}
if (c.get("president") != null)
club.setPresident(c.get("president").toString());
if (c.get("address") != null)

View File

@@ -0,0 +1,108 @@
package org.example;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.firebase.FirebaseApp;
import com.google.firebase.FirebaseOptions;
import com.google.firebase.database.DatabaseReference;
import com.google.firebase.database.FirebaseDatabase;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.example.models.News;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import java.io.FileInputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class NewsScraper {
public static void main(String[] args) {
String caminhoChave = "service-account.json";
String urlDatabase = "https://vdcscore-default-rtdb.firebaseio.com/";
try {
System.out.println("--- A CONECTAR AO FIREBASE ---");
if (FirebaseApp.getApps().isEmpty()) {
FileInputStream serviceAccount = new FileInputStream(caminhoChave);
FirebaseOptions options = FirebaseOptions.builder()
.setCredentials(GoogleCredentials.fromStream(serviceAccount))
.setDatabaseUrl(urlDatabase)
.build();
FirebaseApp.initializeApp(options);
}
} catch (Exception e) {
System.err.println("ERRO FIREBASE: Verifica o service-account.json!");
e.printStackTrace();
return;
}
Gson gson = new Gson();
System.out.println("\n--- A EXTRAIR NOTÍCIAS ---");
try {
// Vamos extrair as primeiras páginas de notícias (ex: 1 a 3)
// Para o resumo na app, talvez apenas a primeira página (mais recentes) baste.
String urlNews = "https://api.afavcd.pt/news/1";
String jsonNews = Jsoup.connect(urlNews)
.ignoreContentType(true)
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
.header("Referer", "https://www.afavcd.pt/")
.header("Origin", "https://www.afavcd.pt")
.method(Connection.Method.GET)
.execute()
.body();
Map<String, Object> responseMap = gson.fromJson(jsonNews, new TypeToken<Map<String, Object>>() {}.getType());
List<Map<String, Object>> rows = (List<Map<String, Object>>) responseMap.get("rows");
List<News> newsList = new ArrayList<>();
if (rows != null) {
for (Map<String, Object> row : rows) {
News n = new News();
if (row.get("newsID") != null) n.setNewsID((int) Double.parseDouble(row.get("newsID").toString()));
if (row.get("title") != null) n.setTitle(row.get("title").toString());
if (row.get("body") != null) n.setBody(row.get("body").toString());
if (row.get("insertDate") != null) n.setInsertDate(row.get("insertDate").toString());
String photo = null;
if (row.get("photoURl") != null) {
photo = row.get("photoURl").toString();
} else if (row.get("photoURL") != null) {
photo = row.get("photoURL").toString();
}
if (photo != null && !photo.isEmpty() && !photo.startsWith("http")) {
photo = "https://api.afavcd.pt" + photo;
}
n.setPhotoURL(photo);
newsList.add(n);
}
}
System.out.println("Encontradas " + newsList.size() + " notícias.");
// Enviar para Firebase
DatabaseReference refNoticias = FirebaseDatabase.getInstance().getReference("noticias");
refNoticias.setValueAsync(newsList);
System.out.println("-> Notícias enviadas para o Firebase.");
} catch (Exception e) {
System.err.println("Erro ao obter notícias.");
e.printStackTrace();
}
try {
System.out.println("\nAguardando confirmação do servidor (5s)...");
Thread.sleep(5000);
System.out.println("Processo concluído!");
System.exit(0);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}

View File

@@ -64,8 +64,13 @@ public class StandingsScraper {
club.setId(clubId);
if (c.get("name") != null)
club.setName(c.get("name").toString());
if (c.get("logoURL") != null)
club.setImageUrl(c.get("logoURL").toString());
if (c.get("logoURL") != null) {
String logoUrl = c.get("logoURL").toString();
if (logoUrl != null && !logoUrl.startsWith("http") && !logoUrl.isEmpty()) {
logoUrl = "https://api.afavcd.pt" + logoUrl;
}
club.setImageUrl(logoUrl);
}
clubesMap.put(clubId, club);
}
@@ -150,13 +155,95 @@ public class StandingsScraper {
posicao++;
}
// Enviar para Firebase
// Enviar para Firebase (Classificações)
DatabaseReference refClassificacoes = FirebaseDatabase.getInstance()
.getReference("classificacoes").child(escalao);
refClassificacoes.setValueAsync(sortedStandings);
System.out.println("-> Classificação de " + escalao + " enviada para o Firebase.");
// Enviar para Firebase (Jornadas)
DatabaseReference refJornadas = FirebaseDatabase.getInstance()
.getReference("jornadas").child(escalao);
for (Map.Entry<String, List<Map<String, Object>>> entry : matchdays.entrySet()) {
String jornadaNumber = entry.getKey();
for (Map<String, Object> match : entry.getValue()) {
try {
int homeId = parseId(match.get("Home"));
int awayId = parseId(match.get("Away"));
// Ignorar jogos com equipas placeholder / de descanso
if (homeId < 0 || awayId < 0) continue;
Club homeClub = clubesMap.get(homeId);
Club awayClub = clubesMap.get(awayId);
String homeName = homeClub != null ? homeClub.getName() : "Equipa " + homeId;
String homeLogo = homeClub != null ? homeClub.getImageUrl() : "";
String awayName = awayClub != null ? awayClub.getName() : "Equipa " + awayId;
String awayLogo = awayClub != null ? awayClub.getImageUrl() : "";
Map<String, Object> matchMap = new HashMap<>();
matchMap.put("home_nome", homeName);
matchMap.put("away_nome", awayName);
matchMap.put("home_logo", homeLogo);
matchMap.put("away_logo", awayLogo);
String homeGoalsStr = String.valueOf(match.get("homeGoals"));
String awayGoalsStr = String.valueOf(match.get("awayGoals"));
boolean played = false;
if (homeGoalsStr != null && !homeGoalsStr.equals("null") && !homeGoalsStr.trim().isEmpty()) {
matchMap.put("home_golos", (int) Double.parseDouble(homeGoalsStr));
played = true;
}
if (awayGoalsStr != null && !awayGoalsStr.equals("null") && !awayGoalsStr.trim().isEmpty()) {
matchMap.put("away_golos", (int) Double.parseDouble(awayGoalsStr));
played = true;
}
matchMap.put("played", played);
Object dateObj = match.get("date");
if (dateObj != null && !String.valueOf(dateObj).equals("null")) {
String dateStr = String.valueOf(dateObj);
if (dateStr.contains("T")) {
String[] parts = dateStr.split("T");
matchMap.put("data", parts[0]);
if (parts[1].length() >= 5) {
matchMap.put("hora", parts[1].substring(0, 5));
}
} else {
matchMap.put("data", dateStr);
}
}
Object fieldObj = match.get("field");
if (fieldObj != null && !String.valueOf(fieldObj).equals("null") && !String.valueOf(fieldObj).trim().isEmpty()) {
matchMap.put("campo", String.valueOf(fieldObj));
}
Object reportUrlObj = match.get("reportURL");
if (reportUrlObj != null && !String.valueOf(reportUrlObj).equals("null") && !String.valueOf(reportUrlObj).trim().isEmpty()) {
matchMap.put("matchReportUrl", "https://api.afavcd.pt" + String.valueOf(reportUrlObj));
}
String jorneyID = String.valueOf(match.get("jorneyID"));
if (jorneyID != null && !jorneyID.equals("null")) {
if (jorneyID.contains(".")) {
jorneyID = jorneyID.substring(0, jorneyID.indexOf("."));
}
refJornadas.child(jornadaNumber).child(jorneyID).setValueAsync(matchMap);
}
} catch (Exception ex) {
System.err.println("Erro ao processar jogo na jornada " + jornadaNumber + ": " + ex.getMessage());
}
}
}
System.out.println("-> Jornadas de " + escalao + " enviadas para o Firebase.");
} catch (Exception e) {
System.err.println("Erro a processar " + escalao);
e.printStackTrace();

View File

@@ -0,0 +1,178 @@
package org.example;
import com.google.auth.oauth2.GoogleCredentials;
import com.google.firebase.FirebaseApp;
import com.google.firebase.FirebaseOptions;
import com.google.firebase.database.DatabaseReference;
import com.google.firebase.database.FirebaseDatabase;
import com.google.gson.Gson;
import com.google.gson.reflect.TypeToken;
import org.example.models.Club;
import org.example.models.TopScorer;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import java.io.FileInputStream;
import java.util.*;
public class TopScorersScraper {
public static void main(String[] args) {
String caminhoChave = "service-account.json";
String urlDatabase = "https://vdcscore-default-rtdb.firebaseio.com/";
try {
System.out.println("--- A CONECTAR AO FIREBASE ---");
if (FirebaseApp.getApps().isEmpty()) {
FileInputStream serviceAccount = new FileInputStream(caminhoChave);
FirebaseOptions options = FirebaseOptions.builder()
.setCredentials(GoogleCredentials.fromStream(serviceAccount))
.setDatabaseUrl(urlDatabase)
.build();
FirebaseApp.initializeApp(options);
}
} catch (Exception e) {
System.err.println("ERRO FIREBASE: Verifica o service-account.json!");
e.printStackTrace();
return;
}
Gson gson = new Gson();
Map<Integer, Club> clubesMap = new HashMap<>();
// 1. Obter todos os Clubes para ter logos e nomes reais
System.out.println("\n--- A EXTRAIR DADOS DOS CLUBES ---");
try {
String urlTodosClubes = "https://api.afavcd.pt/teams";
String jsonTodosClubes = Jsoup.connect(urlTodosClubes)
.ignoreContentType(true)
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
.header("Referer", "https://www.afavcd.pt/")
.header("Origin", "https://www.afavcd.pt")
.maxBodySize(0)
.timeout(30000)
.method(Connection.Method.GET)
.execute()
.body();
List<Map<String, Object>> dadosClubes = gson.fromJson(jsonTodosClubes,
new TypeToken<List<Map<String, Object>>>() {}.getType());
for (Map<String, Object> c : dadosClubes) {
if (c.get("teamID") != null) {
int clubId = (int) Double.parseDouble(c.get("teamID").toString());
Club club = new Club();
club.setId(clubId);
if (c.get("name") != null) club.setName(c.get("name").toString());
if (c.get("logoURL") != null) {
String logoUrl = c.get("logoURL").toString();
if (logoUrl != null && !logoUrl.startsWith("http") && !logoUrl.isEmpty()) {
logoUrl = "https://api.afavcd.pt" + logoUrl;
}
club.setImageUrl(logoUrl);
}
clubesMap.put(clubId, club);
}
}
System.out.println("Encontrados " + clubesMap.size() + " clubes.");
} catch (Exception e) {
System.err.println("Erro ao obter lista de clubes.");
e.printStackTrace();
return;
}
int[] modalidades = { 1, 3 }; // 1: Seniores, 3: Juniores
String[] nomesEscaloes = { "seniores", "juniores" };
for (int i = 0; i < modalidades.length; i++) {
int modalidade = modalidades[i];
String escalao = nomesEscaloes[i];
System.out.println("\n--- A EXTRAIR MARCADORES PARA: " + escalao.toUpperCase() + " ---");
String urlDiscipline = "https://api.afavcd.pt/teams/modality/" + modalidade + "/season/33/discipline";
try {
String jsonDiscipline = Jsoup.connect(urlDiscipline)
.ignoreContentType(true)
.userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64)")
.header("Referer", "https://www.afavcd.pt/")
.header("Origin", "https://www.afavcd.pt")
.maxBodySize(0)
.timeout(60000)
.method(Connection.Method.GET)
.execute()
.body();
Map<String, Object> root = gson.fromJson(jsonDiscipline, new TypeToken<Map<String, Object>>() {}.getType());
Map<String, List<Map<String, Object>>> teams = (Map<String, List<Map<String, Object>>>) root.get("teams");
List<TopScorer> scorersList = new ArrayList<>();
if (teams != null) {
for (List<Map<String, Object>> players : teams.values()) {
for (Map<String, Object> player : players) {
String playerName = (String) player.get("fullName");
String playerPhoto = (String) player.get("photoURl");
int clubId = (int) Double.parseDouble(player.get("teamID").toString());
Club club = clubesMap.get(clubId);
String clubName = club != null ? club.getName() : (String) player.get("name");
String clubLogo = club != null ? club.getImageUrl() : "";
List<Map<String, Object>> jorneys = (List<Map<String, Object>>) player.get("jorneys");
int totalGoals = 0;
if (jorneys != null) {
for (Map<String, Object> j : jorneys) {
Object goalsObj = j.get("totalGoals");
if (goalsObj != null && !goalsObj.toString().isEmpty()) {
try {
totalGoals += (int) Double.parseDouble(goalsObj.toString());
} catch (Exception ignored) {}
}
}
}
if (totalGoals > 0) {
TopScorer ts = new TopScorer();
ts.setPlayerName(playerName);
ts.setPlayerPhoto(playerPhoto != null && !playerPhoto.startsWith("http") ? "https://api.afavcd.pt" + playerPhoto : playerPhoto);
ts.setClubName(clubName);
ts.setClubLogo(clubLogo);
ts.setGoals(totalGoals);
scorersList.add(ts);
}
}
}
}
// Ordenar por golos
Collections.sort(scorersList);
// Definir posições e limitar aos top 20 (opcional, mas comum)
for (int j = 0; j < scorersList.size(); j++) {
scorersList.get(j).setPosition(j + 1);
}
// Enviar para Firebase
DatabaseReference refMarcadores = FirebaseDatabase.getInstance()
.getReference("marcadores").child(escalao);
refMarcadores.setValueAsync(scorersList);
System.out.println("-> " + scorersList.size() + " marcadores de " + escalao + " enviados para o Firebase.");
} catch (Exception e) {
System.err.println("Erro ao processar marcadores de " + escalao);
e.printStackTrace();
}
}
try {
System.out.println("\nAguardando confirmação do servidor (5s)...");
Thread.sleep(5000);
System.out.println("Processo concluído!");
System.exit(0);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}

View File

@@ -0,0 +1,52 @@
package org.example.models;
public class News {
private int newsID;
private String title;
private String body;
private String insertDate;
private String photoURL;
public News() {
}
public int getNewsID() {
return newsID;
}
public void setNewsID(int newsID) {
this.newsID = newsID;
}
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getBody() {
return body;
}
public void setBody(String body) {
this.body = body;
}
public String getInsertDate() {
return insertDate;
}
public void setInsertDate(String insertDate) {
this.insertDate = insertDate;
}
public String getPhotoURL() {
return photoURL;
}
public void setPhotoURL(String photoURL) {
this.photoURL = photoURL;
}
}

View File

@@ -0,0 +1,70 @@
package org.example.models;
import java.io.Serializable;
public class TopScorer implements Serializable, Comparable<TopScorer> {
private String playerName;
private String playerPhoto;
private String clubName;
private String clubLogo;
private int goals;
private int position;
public TopScorer() {
}
public String getPlayerName() {
return playerName;
}
public void setPlayerName(String playerName) {
this.playerName = playerName;
}
public String getPlayerPhoto() {
return playerPhoto;
}
public void setPlayerPhoto(String playerPhoto) {
this.playerPhoto = playerPhoto;
}
public String getClubName() {
return clubName;
}
public void setClubName(String clubName) {
this.clubName = clubName;
}
public String getClubLogo() {
return clubLogo;
}
public void setClubLogo(String clubLogo) {
this.clubLogo = clubLogo;
}
public int getGoals() {
return goals;
}
public void setGoals(int goals) {
this.goals = goals;
}
public int getPosition() {
return position;
}
public void setPosition(int position) {
this.position = position;
}
@Override
public int compareTo(TopScorer other) {
// Ordenar por golos decrescente
return Integer.compare(other.getGoals(), this.getGoals());
}
}