diff --git a/.idea/.name b/.idea/.name new file mode 100644 index 0000000..af11754 --- /dev/null +++ b/.idea/.name @@ -0,0 +1 @@ +testeScraper \ No newline at end of file diff --git a/src/main/java/org/example/NewsScraper.java b/src/main/java/org/example/NewsScraper.java new file mode 100644 index 0000000..fbee733 --- /dev/null +++ b/src/main/java/org/example/NewsScraper.java @@ -0,0 +1,108 @@ +package org.example; + +import com.google.auth.oauth2.GoogleCredentials; +import com.google.firebase.FirebaseApp; +import com.google.firebase.FirebaseOptions; +import com.google.firebase.database.DatabaseReference; +import com.google.firebase.database.FirebaseDatabase; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import org.example.models.News; +import org.jsoup.Connection; +import org.jsoup.Jsoup; + +import java.io.FileInputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class NewsScraper { + public static void main(String[] args) { + + String caminhoChave = "service-account.json"; + String urlDatabase = "https://vdcscore-default-rtdb.firebaseio.com/"; + + try { + System.out.println("--- A CONECTAR AO FIREBASE ---"); + if (FirebaseApp.getApps().isEmpty()) { + FileInputStream serviceAccount = new FileInputStream(caminhoChave); + FirebaseOptions options = FirebaseOptions.builder() + .setCredentials(GoogleCredentials.fromStream(serviceAccount)) + .setDatabaseUrl(urlDatabase) + .build(); + FirebaseApp.initializeApp(options); + } + } catch (Exception e) { + System.err.println("ERRO FIREBASE: Verifica o service-account.json!"); + e.printStackTrace(); + return; + } + + Gson gson = new Gson(); + System.out.println("\n--- A EXTRAIR NOTÍCIAS ---"); + + try { + // Vamos extrair as primeiras páginas de notícias (ex: 1 a 3) + // Para o resumo na app, talvez apenas a primeira página (mais recentes) baste. + String urlNews = "https://api.afavcd.pt/news/1"; + + String jsonNews = Jsoup.connect(urlNews) + .ignoreContentType(true) + .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64)") + .header("Referer", "https://www.afavcd.pt/") + .header("Origin", "https://www.afavcd.pt") + .method(Connection.Method.GET) + .execute() + .body(); + + Map responseMap = gson.fromJson(jsonNews, new TypeToken>() {}.getType()); + List> rows = (List>) responseMap.get("rows"); + + List newsList = new ArrayList<>(); + + if (rows != null) { + for (Map row : rows) { + News n = new News(); + if (row.get("newsID") != null) n.setNewsID((int) Double.parseDouble(row.get("newsID").toString())); + if (row.get("title") != null) n.setTitle(row.get("title").toString()); + if (row.get("body") != null) n.setBody(row.get("body").toString()); + if (row.get("insertDate") != null) n.setInsertDate(row.get("insertDate").toString()); + + String photo = null; + if (row.get("photoURl") != null) { + photo = row.get("photoURl").toString(); + } else if (row.get("photoURL") != null) { + photo = row.get("photoURL").toString(); + } + + if (photo != null && !photo.isEmpty() && !photo.startsWith("http")) { + photo = "https://api.afavcd.pt" + photo; + } + n.setPhotoURL(photo); + + newsList.add(n); + } + } + + System.out.println("Encontradas " + newsList.size() + " notícias."); + + // Enviar para Firebase + DatabaseReference refNoticias = FirebaseDatabase.getInstance().getReference("noticias"); + refNoticias.setValueAsync(newsList); + System.out.println("-> Notícias enviadas para o Firebase."); + + } catch (Exception e) { + System.err.println("Erro ao obter notícias."); + e.printStackTrace(); + } + + try { + System.out.println("\nAguardando confirmação do servidor (5s)..."); + Thread.sleep(5000); + System.out.println("Processo concluído!"); + System.exit(0); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } +} diff --git a/src/main/java/org/example/TopScorersScraper.java b/src/main/java/org/example/TopScorersScraper.java new file mode 100644 index 0000000..6786e0e --- /dev/null +++ b/src/main/java/org/example/TopScorersScraper.java @@ -0,0 +1,178 @@ +package org.example; + +import com.google.auth.oauth2.GoogleCredentials; +import com.google.firebase.FirebaseApp; +import com.google.firebase.FirebaseOptions; +import com.google.firebase.database.DatabaseReference; +import com.google.firebase.database.FirebaseDatabase; +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import org.example.models.Club; +import org.example.models.TopScorer; +import org.jsoup.Connection; +import org.jsoup.Jsoup; + +import java.io.FileInputStream; +import java.util.*; + +public class TopScorersScraper { + + public static void main(String[] args) { + String caminhoChave = "service-account.json"; + String urlDatabase = "https://vdcscore-default-rtdb.firebaseio.com/"; + + try { + System.out.println("--- A CONECTAR AO FIREBASE ---"); + if (FirebaseApp.getApps().isEmpty()) { + FileInputStream serviceAccount = new FileInputStream(caminhoChave); + FirebaseOptions options = FirebaseOptions.builder() + .setCredentials(GoogleCredentials.fromStream(serviceAccount)) + .setDatabaseUrl(urlDatabase) + .build(); + FirebaseApp.initializeApp(options); + } + } catch (Exception e) { + System.err.println("ERRO FIREBASE: Verifica o service-account.json!"); + e.printStackTrace(); + return; + } + + Gson gson = new Gson(); + Map clubesMap = new HashMap<>(); + + // 1. Obter todos os Clubes para ter logos e nomes reais + System.out.println("\n--- A EXTRAIR DADOS DOS CLUBES ---"); + try { + String urlTodosClubes = "https://api.afavcd.pt/teams"; + String jsonTodosClubes = Jsoup.connect(urlTodosClubes) + .ignoreContentType(true) + .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64)") + .header("Referer", "https://www.afavcd.pt/") + .header("Origin", "https://www.afavcd.pt") + .maxBodySize(0) + .timeout(30000) + .method(Connection.Method.GET) + .execute() + .body(); + + List> dadosClubes = gson.fromJson(jsonTodosClubes, + new TypeToken>>() {}.getType()); + + for (Map c : dadosClubes) { + if (c.get("teamID") != null) { + int clubId = (int) Double.parseDouble(c.get("teamID").toString()); + Club club = new Club(); + club.setId(clubId); + if (c.get("name") != null) club.setName(c.get("name").toString()); + if (c.get("logoURL") != null) { + String logoUrl = c.get("logoURL").toString(); + if (logoUrl != null && !logoUrl.startsWith("http") && !logoUrl.isEmpty()) { + logoUrl = "https://api.afavcd.pt" + logoUrl; + } + club.setImageUrl(logoUrl); + } + clubesMap.put(clubId, club); + } + } + System.out.println("Encontrados " + clubesMap.size() + " clubes."); + } catch (Exception e) { + System.err.println("Erro ao obter lista de clubes."); + e.printStackTrace(); + return; + } + + int[] modalidades = { 1, 3 }; // 1: Seniores, 3: Juniores + String[] nomesEscaloes = { "seniores", "juniores" }; + + for (int i = 0; i < modalidades.length; i++) { + int modalidade = modalidades[i]; + String escalao = nomesEscaloes[i]; + + System.out.println("\n--- A EXTRAIR MARCADORES PARA: " + escalao.toUpperCase() + " ---"); + + String urlDiscipline = "https://api.afavcd.pt/teams/modality/" + modalidade + "/season/33/discipline"; + try { + String jsonDiscipline = Jsoup.connect(urlDiscipline) + .ignoreContentType(true) + .userAgent("Mozilla/5.0 (Windows NT 10.0; Win64; x64)") + .header("Referer", "https://www.afavcd.pt/") + .header("Origin", "https://www.afavcd.pt") + .maxBodySize(0) + .timeout(60000) + .method(Connection.Method.GET) + .execute() + .body(); + + Map root = gson.fromJson(jsonDiscipline, new TypeToken>() {}.getType()); + Map>> teams = (Map>>) root.get("teams"); + + List scorersList = new ArrayList<>(); + + if (teams != null) { + for (List> players : teams.values()) { + for (Map player : players) { + String playerName = (String) player.get("fullName"); + String playerPhoto = (String) player.get("photoURl"); + int clubId = (int) Double.parseDouble(player.get("teamID").toString()); + + Club club = clubesMap.get(clubId); + String clubName = club != null ? club.getName() : (String) player.get("name"); + String clubLogo = club != null ? club.getImageUrl() : ""; + + List> jorneys = (List>) player.get("jorneys"); + int totalGoals = 0; + if (jorneys != null) { + for (Map j : jorneys) { + Object goalsObj = j.get("totalGoals"); + if (goalsObj != null && !goalsObj.toString().isEmpty()) { + try { + totalGoals += (int) Double.parseDouble(goalsObj.toString()); + } catch (Exception ignored) {} + } + } + } + + if (totalGoals > 0) { + TopScorer ts = new TopScorer(); + ts.setPlayerName(playerName); + ts.setPlayerPhoto(playerPhoto != null && !playerPhoto.startsWith("http") ? "https://api.afavcd.pt" + playerPhoto : playerPhoto); + ts.setClubName(clubName); + ts.setClubLogo(clubLogo); + ts.setGoals(totalGoals); + scorersList.add(ts); + } + } + } + } + + // Ordenar por golos + Collections.sort(scorersList); + + // Definir posições e limitar aos top 20 (opcional, mas comum) + for (int j = 0; j < scorersList.size(); j++) { + scorersList.get(j).setPosition(j + 1); + } + + // Enviar para Firebase + DatabaseReference refMarcadores = FirebaseDatabase.getInstance() + .getReference("marcadores").child(escalao); + + refMarcadores.setValueAsync(scorersList); + System.out.println("-> " + scorersList.size() + " marcadores de " + escalao + " enviados para o Firebase."); + + } catch (Exception e) { + System.err.println("Erro ao processar marcadores de " + escalao); + e.printStackTrace(); + } + } + + try { + System.out.println("\nAguardando confirmação do servidor (5s)..."); + Thread.sleep(5000); + System.out.println("Processo concluído!"); + System.exit(0); + } catch (InterruptedException e) { + e.printStackTrace(); + } + } +} diff --git a/src/main/java/org/example/models/News.java b/src/main/java/org/example/models/News.java new file mode 100644 index 0000000..de729de --- /dev/null +++ b/src/main/java/org/example/models/News.java @@ -0,0 +1,52 @@ +package org.example.models; + +public class News { + private int newsID; + private String title; + private String body; + private String insertDate; + private String photoURL; + + public News() { + } + + public int getNewsID() { + return newsID; + } + + public void setNewsID(int newsID) { + this.newsID = newsID; + } + + public String getTitle() { + return title; + } + + public void setTitle(String title) { + this.title = title; + } + + public String getBody() { + return body; + } + + public void setBody(String body) { + this.body = body; + } + + public String getInsertDate() { + return insertDate; + } + + public void setInsertDate(String insertDate) { + this.insertDate = insertDate; + } + + public String getPhotoURL() { + return photoURL; + } + + public void setPhotoURL(String photoURL) { + this.photoURL = photoURL; + } +} diff --git a/src/main/java/org/example/models/TopScorer.java b/src/main/java/org/example/models/TopScorer.java new file mode 100644 index 0000000..585e6b2 --- /dev/null +++ b/src/main/java/org/example/models/TopScorer.java @@ -0,0 +1,70 @@ +package org.example.models; + +import java.io.Serializable; + +public class TopScorer implements Serializable, Comparable { + + private String playerName; + private String playerPhoto; + private String clubName; + private String clubLogo; + private int goals; + private int position; + + public TopScorer() { + } + + public String getPlayerName() { + return playerName; + } + + public void setPlayerName(String playerName) { + this.playerName = playerName; + } + + public String getPlayerPhoto() { + return playerPhoto; + } + + public void setPlayerPhoto(String playerPhoto) { + this.playerPhoto = playerPhoto; + } + + public String getClubName() { + return clubName; + } + + public void setClubName(String clubName) { + this.clubName = clubName; + } + + public String getClubLogo() { + return clubLogo; + } + + public void setClubLogo(String clubLogo) { + this.clubLogo = clubLogo; + } + + public int getGoals() { + return goals; + } + + public void setGoals(int goals) { + this.goals = goals; + } + + public int getPosition() { + return position; + } + + public void setPosition(int position) { + this.position = position; + } + + @Override + public int compareTo(TopScorer other) { + // Ordenar por golos decrescente + return Integer.compare(other.getGoals(), this.getGoals()); + } +}