136 lines
5.9 KiB
Dart
136 lines
5.9 KiB
Dart
|
|
import 'dart:convert';
|
|
import 'dart:io';
|
|
|
|
import 'package:capstone_project/providers/settings.dart';
|
|
import 'package:capstone_project/utils/agrigator.dart';
|
|
import 'package:capstone_project/widgets/navbar.dart';
|
|
import 'package:go_router/go_router.dart';
|
|
import 'package:shadcn_flutter/shadcn_flutter.dart';
|
|
|
|
class HomePage extends StatelessWidget {
|
|
|
|
static GoRoute route = GoRoute(
|
|
path: "/",
|
|
builder: (context, state) => HomePage()
|
|
);
|
|
|
|
@override
|
|
Widget build(BuildContext context) {
|
|
// TODO: implement build
|
|
return Scaffold(
|
|
headers: [
|
|
AppBar()
|
|
],
|
|
footers: [
|
|
ProjNavBar(
|
|
currentPage: "home",
|
|
)
|
|
],
|
|
child: Center(
|
|
child: Column(
|
|
mainAxisSize: MainAxisSize.min,
|
|
children: [
|
|
|
|
Text(
|
|
"Nothing here is final yet!",
|
|
).h1,
|
|
|
|
Gap(16),
|
|
|
|
Button.primary(
|
|
onPressed: () async {
|
|
print("Aggregating feeds...");
|
|
SettingsProvider settings = SettingsProvider.of(context);
|
|
|
|
// Fetch all feeds
|
|
List<Uri> feedUris = settings.feeds.map((feed) => Uri.parse(feed.url)).toList();
|
|
List<FeedItem> aggregatedItems = await fetchFeeds(feedUris);
|
|
|
|
// Save it to a file before generating embeddings
|
|
String agregatedJson = JsonEncoder.withIndent(' ').convert(aggregatedItems);
|
|
File agregatedJsonFile = File("${settings.applicationStorageLocation}/aggregated_feed.json");
|
|
await agregatedJsonFile.writeAsString(agregatedJson);
|
|
print("Aggregated feed saved to ${agregatedJsonFile.path}");
|
|
|
|
// Generate embeddings for all items
|
|
print("Generating embeddings for ${aggregatedItems.length} items...");
|
|
List<FeedItem> enrichedItems = [...aggregatedItems];
|
|
await generateEmbeddings(enrichedItems, settings.openAIApiKey);
|
|
|
|
// Save it to a file in the application storage location
|
|
String enrichedJson = JsonEncoder.withIndent(' ').convert(enrichedItems);
|
|
final file = File("${settings.applicationStorageLocation}/enriched_aggregated_feed.json");
|
|
await file.writeAsString(enrichedJson);
|
|
print("Enriched aggregated feed saved to ${file.path}");
|
|
|
|
// Filter out irrelevant items
|
|
print("Filtering relevant items...");
|
|
await generateKeywordEmbeddings(settings.openAIApiKey);
|
|
List<FeedItem> relevantItems = [...aggregatedItems]..removeWhere((item) => !isFeedItemRelevant(item));
|
|
String relevantJson = JsonEncoder.withIndent(' ').convert(relevantItems);
|
|
final fileRelevant = File("${settings.applicationStorageLocation}/relevant_aggregated_feed.json");
|
|
await fileRelevant.writeAsString(relevantJson);
|
|
print("Cut down from ${aggregatedItems.length} to ${relevantItems.length} relevant items.");
|
|
print("Relevant aggregated feed saved to ${fileRelevant.path}");
|
|
|
|
// For human readability, save a version without embeddings
|
|
List<FeedItem> readableItems = relevantItems.map((item) {
|
|
return FeedItem(
|
|
title: item.title,
|
|
link: item.link,
|
|
description: item.description,
|
|
);
|
|
}).toList();
|
|
|
|
String readableJson = JsonEncoder.withIndent(' ').convert(readableItems);
|
|
final fileReadable = File("${settings.applicationStorageLocation}/readable_relevant_aggregated_feed.json");
|
|
await fileReadable.writeAsString(readableJson);
|
|
print("Readable relevant aggregated feed saved to ${fileReadable.path}");
|
|
|
|
// Group by event
|
|
print("Grouping feed items by event...");
|
|
List<List<FeedItem>> groupedItems = groupFeedItemsByEvent(relevantItems);
|
|
List<List<Map<String, dynamic>>> groupedItemsJson = groupedItems.map((group) {
|
|
return group.map((item) => item.toJson()).toList();
|
|
}).toList();
|
|
String groupedJson = JsonEncoder.withIndent(' ').convert(groupedItemsJson);
|
|
final fileGrouped = File("${settings.applicationStorageLocation}/grouped_relevant_aggregated_feed.json");
|
|
await fileGrouped.writeAsString(groupedJson);
|
|
print("Grouped relevant aggregated feed saved to ${fileGrouped.path}");
|
|
|
|
// For human readability, save a version without embeddings
|
|
List<List<FeedItem>> readableGroupedItems = groupedItems.map((group) {
|
|
return group.map((item) {
|
|
return FeedItem(
|
|
title: item.title,
|
|
link: item.link,
|
|
description: item.description,
|
|
);
|
|
}).toList();
|
|
}).toList();
|
|
// Sort groups by size descending
|
|
readableGroupedItems.sort((a, b) => b.length.compareTo(a.length));
|
|
List<List<Map<String, dynamic>>> readableGroupedItemsJson = readableGroupedItems.map((group) {
|
|
return group.map((item) => item.toJson()).toList();
|
|
}).toList();
|
|
String readableGroupedJson = JsonEncoder.withIndent(' ').convert(readableGroupedItemsJson);
|
|
final fileReadableGrouped = File("${settings.applicationStorageLocation}/readable_grouped_relevant_aggregated_feed.json");
|
|
await fileReadableGrouped.writeAsString(readableGroupedJson);
|
|
print("Readable grouped relevant aggregated feed saved to ${fileReadableGrouped.path}");
|
|
|
|
},
|
|
child: Text(
|
|
"Aggregate your feeds"
|
|
),
|
|
)
|
|
|
|
|
|
|
|
],
|
|
),
|
|
),
|
|
);
|
|
}
|
|
|
|
} |