Add feed aggregation and embedding generation features
This commit is contained in:
ImBenji
@@ -1,4 +1,9 @@
|
||||
|
||||
import 'dart:convert';
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:capstone_project/providers/settings.dart';
|
||||
import 'package:capstone_project/utils/agrigator.dart';
|
||||
import 'package:capstone_project/widgets/navbar.dart';
|
||||
import 'package:go_router/go_router.dart';
|
||||
import 'package:shadcn_flutter/shadcn_flutter.dart';
|
||||
@@ -22,12 +27,108 @@ class HomePage extends StatelessWidget {
|
||||
currentPage: "home",
|
||||
)
|
||||
],
|
||||
child: Column(
|
||||
children: [
|
||||
child: Center(
|
||||
child: Column(
|
||||
mainAxisSize: MainAxisSize.min,
|
||||
children: [
|
||||
|
||||
Text(
|
||||
"Nothing here is final yet!",
|
||||
).h1,
|
||||
|
||||
Gap(16),
|
||||
|
||||
Button.primary(
|
||||
onPressed: () async {
|
||||
print("Aggregating feeds...");
|
||||
SettingsProvider settings = SettingsProvider.of(context);
|
||||
|
||||
// Fetch all feeds
|
||||
List<Uri> feedUris = settings.feeds.map((feed) => Uri.parse(feed.url)).toList();
|
||||
List<FeedItem> aggregatedItems = await fetchFeeds(feedUris);
|
||||
|
||||
// Save it to a file before generating embeddings
|
||||
String agregatedJson = JsonEncoder.withIndent(' ').convert(aggregatedItems);
|
||||
File agregatedJsonFile = File("${settings.applicationStorageLocation}/aggregated_feed.json");
|
||||
await agregatedJsonFile.writeAsString(agregatedJson);
|
||||
print("Aggregated feed saved to ${agregatedJsonFile.path}");
|
||||
|
||||
// Generate embeddings for all items
|
||||
print("Generating embeddings for ${aggregatedItems.length} items...");
|
||||
List<FeedItem> enrichedItems = [...aggregatedItems];
|
||||
await generateEmbeddings(enrichedItems, settings.openAIApiKey);
|
||||
|
||||
// Save it to a file in the application storage location
|
||||
String enrichedJson = JsonEncoder.withIndent(' ').convert(enrichedItems);
|
||||
final file = File("${settings.applicationStorageLocation}/enriched_aggregated_feed.json");
|
||||
await file.writeAsString(enrichedJson);
|
||||
print("Enriched aggregated feed saved to ${file.path}");
|
||||
|
||||
// Filter out irrelevant items
|
||||
print("Filtering relevant items...");
|
||||
await generateKeywordEmbeddings(settings.openAIApiKey);
|
||||
List<FeedItem> relevantItems = [...aggregatedItems]..removeWhere((item) => !isFeedItemRelevant(item));
|
||||
String relevantJson = JsonEncoder.withIndent(' ').convert(relevantItems);
|
||||
final fileRelevant = File("${settings.applicationStorageLocation}/relevant_aggregated_feed.json");
|
||||
await fileRelevant.writeAsString(relevantJson);
|
||||
print("Cut down from ${aggregatedItems.length} to ${relevantItems.length} relevant items.");
|
||||
print("Relevant aggregated feed saved to ${fileRelevant.path}");
|
||||
|
||||
// For human readability, save a version without embeddings
|
||||
List<FeedItem> readableItems = relevantItems.map((item) {
|
||||
return FeedItem(
|
||||
title: item.title,
|
||||
link: item.link,
|
||||
description: item.description,
|
||||
);
|
||||
}).toList();
|
||||
|
||||
String readableJson = JsonEncoder.withIndent(' ').convert(readableItems);
|
||||
final fileReadable = File("${settings.applicationStorageLocation}/readable_relevant_aggregated_feed.json");
|
||||
await fileReadable.writeAsString(readableJson);
|
||||
print("Readable relevant aggregated feed saved to ${fileReadable.path}");
|
||||
|
||||
// Group by event
|
||||
print("Grouping feed items by event...");
|
||||
List<List<FeedItem>> groupedItems = groupFeedItemsByEvent(relevantItems);
|
||||
List<List<Map<String, dynamic>>> groupedItemsJson = groupedItems.map((group) {
|
||||
return group.map((item) => item.toJson()).toList();
|
||||
}).toList();
|
||||
String groupedJson = JsonEncoder.withIndent(' ').convert(groupedItemsJson);
|
||||
final fileGrouped = File("${settings.applicationStorageLocation}/grouped_relevant_aggregated_feed.json");
|
||||
await fileGrouped.writeAsString(groupedJson);
|
||||
print("Grouped relevant aggregated feed saved to ${fileGrouped.path}");
|
||||
|
||||
// For human readability, save a version without embeddings
|
||||
List<List<FeedItem>> readableGroupedItems = groupedItems.map((group) {
|
||||
return group.map((item) {
|
||||
return FeedItem(
|
||||
title: item.title,
|
||||
link: item.link,
|
||||
description: item.description,
|
||||
);
|
||||
}).toList();
|
||||
}).toList();
|
||||
// Sort groups by size descending
|
||||
readableGroupedItems.sort((a, b) => b.length.compareTo(a.length));
|
||||
List<List<Map<String, dynamic>>> readableGroupedItemsJson = readableGroupedItems.map((group) {
|
||||
return group.map((item) => item.toJson()).toList();
|
||||
}).toList();
|
||||
String readableGroupedJson = JsonEncoder.withIndent(' ').convert(readableGroupedItemsJson);
|
||||
final fileReadableGrouped = File("${settings.applicationStorageLocation}/readable_grouped_relevant_aggregated_feed.json");
|
||||
await fileReadableGrouped.writeAsString(readableGroupedJson);
|
||||
print("Readable grouped relevant aggregated feed saved to ${fileReadableGrouped.path}");
|
||||
|
||||
},
|
||||
child: Text(
|
||||
"Aggregate your feeds"
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
],
|
||||
],
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,18 +1,65 @@
|
||||
|
||||
import 'package:capstone_project/providers/settings.dart';
|
||||
import 'package:capstone_project/widgets/navbar.dart';
|
||||
import 'package:file_picker/file_picker.dart';
|
||||
import 'package:go_router/go_router.dart';
|
||||
import 'package:path_provider/path_provider.dart';
|
||||
import 'package:shadcn_flutter/shadcn_flutter.dart';
|
||||
|
||||
class SettingsPage extends StatelessWidget {
|
||||
class SettingsPage extends StatefulWidget {
|
||||
|
||||
static GoRoute route = GoRoute(
|
||||
path: "/settings",
|
||||
builder: (context, state) => SettingsPage()
|
||||
);
|
||||
|
||||
@override
|
||||
State<SettingsPage> createState() => _SettingsPageState();
|
||||
}
|
||||
|
||||
class _SettingsPageState extends State<SettingsPage> {
|
||||
|
||||
bool _isLoading = true;
|
||||
|
||||
String apiKey = "";
|
||||
List<Feed> feeds = [];
|
||||
String appStorageLocation = "";
|
||||
|
||||
@override
|
||||
void initState() {
|
||||
// TODO: implement initState
|
||||
super.initState();
|
||||
|
||||
loadPage();
|
||||
}
|
||||
|
||||
void loadPage() async {
|
||||
SettingsProvider settings = SettingsProvider.of(context);
|
||||
|
||||
setState(() {
|
||||
_isLoading = true;
|
||||
});
|
||||
|
||||
apiKey = settings.openAIApiKey;
|
||||
feeds = settings.feeds;
|
||||
appStorageLocation = settings.applicationStorageLocation;
|
||||
|
||||
setState(() {
|
||||
_isLoading = false;
|
||||
});
|
||||
}
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
// TODO: implement build
|
||||
|
||||
if (_isLoading) {
|
||||
return Scaffold(
|
||||
child: Center(
|
||||
child: CircularProgressIndicator(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return Scaffold(
|
||||
headers: [
|
||||
AppBar(
|
||||
@@ -24,58 +71,377 @@ class SettingsPage extends StatelessWidget {
|
||||
currentPage: "settings",
|
||||
)
|
||||
],
|
||||
child: Column(
|
||||
children: [
|
||||
|
||||
SizedBox(
|
||||
width: double.infinity,
|
||||
child: Card(
|
||||
child: Column(
|
||||
crossAxisAlignment: CrossAxisAlignment.start,
|
||||
children: [
|
||||
|
||||
Text(
|
||||
"API Key"
|
||||
).extraBold,
|
||||
|
||||
const SizedBox(height: 16),
|
||||
|
||||
TextField(
|
||||
|
||||
child: Center(
|
||||
child: SizedBox(
|
||||
width: 600,
|
||||
child: SingleChildScrollView(
|
||||
child: Column(
|
||||
crossAxisAlignment: CrossAxisAlignment.start,
|
||||
children: [
|
||||
|
||||
Text(
|
||||
"Open AI"
|
||||
).h4,
|
||||
|
||||
Gap(12),
|
||||
|
||||
Text(
|
||||
"API Key"
|
||||
).small.normal,
|
||||
|
||||
Gap(8),
|
||||
|
||||
TextField(
|
||||
placeholder: Text(
|
||||
"Enter your OpenAI API key"
|
||||
),
|
||||
|
||||
initialValue: apiKey.substring(0, 8) + "xxxxxx (Redacted for security)" ,
|
||||
onChanged: (value) {
|
||||
SettingsProvider settings = SettingsProvider.of(context);
|
||||
|
||||
settings.setOpenAIApiKey(value);
|
||||
},
|
||||
),
|
||||
|
||||
Gap(8),
|
||||
|
||||
if (apiKey.isEmpty)
|
||||
DestructiveBadge(
|
||||
child: Text(
|
||||
"API key is required to use AI features."
|
||||
),
|
||||
),
|
||||
|
||||
Gap(16),
|
||||
|
||||
Divider(),
|
||||
|
||||
Gap(16),
|
||||
|
||||
Row(
|
||||
children: [
|
||||
Expanded(
|
||||
child: Column(
|
||||
crossAxisAlignment: CrossAxisAlignment.start,
|
||||
children: [
|
||||
Text(
|
||||
"News Feeds"
|
||||
).h4,
|
||||
|
||||
Gap(4),
|
||||
|
||||
Text(
|
||||
"Manage your RSS news feeds."
|
||||
).muted.small,
|
||||
],
|
||||
),
|
||||
),
|
||||
|
||||
Button.outline(
|
||||
child: Text(
|
||||
"Add Feed"
|
||||
),
|
||||
onPressed: () {
|
||||
showDialog(
|
||||
context: context,
|
||||
barrierDismissible: false,
|
||||
builder: (context) => AddFeedDialog()
|
||||
).then((value) {
|
||||
loadPage();
|
||||
});
|
||||
},
|
||||
)
|
||||
],
|
||||
),
|
||||
|
||||
Gap(24),
|
||||
|
||||
for (Feed feed in feeds) ...[
|
||||
|
||||
Row(
|
||||
children: [
|
||||
Expanded(
|
||||
child: Text(
|
||||
feed.title.isNotEmpty ? feed.title : feed.url,
|
||||
style: TextStyle(
|
||||
decoration: feed.enabled ? TextDecoration.none : TextDecoration.lineThrough
|
||||
),
|
||||
),
|
||||
),
|
||||
|
||||
Builder(
|
||||
builder: (context2) {
|
||||
return GhostButton(
|
||||
density: ButtonDensity.dense,
|
||||
child: Icon(
|
||||
LucideIcons.ellipsis,
|
||||
|
||||
),
|
||||
onPressed: () {
|
||||
|
||||
|
||||
|
||||
showDropdown(
|
||||
context: context2,
|
||||
builder: (_) {
|
||||
return DropdownMenu(
|
||||
children: [
|
||||
MenuButton(
|
||||
child: Text(
|
||||
"Edit"
|
||||
),
|
||||
trailing: Icon(
|
||||
LucideIcons.filePen
|
||||
),
|
||||
onPressed: (_) {
|
||||
// Navigator.of(context).pop();
|
||||
|
||||
showDialog(
|
||||
context: context,
|
||||
barrierDismissible: false,
|
||||
builder: (context) => AddFeedDialog(
|
||||
existingFeed: feed,
|
||||
)
|
||||
).then((value) {
|
||||
loadPage();
|
||||
});
|
||||
|
||||
},
|
||||
)
|
||||
],
|
||||
);
|
||||
}
|
||||
);
|
||||
},
|
||||
);
|
||||
}
|
||||
),
|
||||
|
||||
Gap(4),
|
||||
|
||||
Switch(
|
||||
value: feed.enabled,
|
||||
onChanged: (value) {
|
||||
SettingsProvider settings = SettingsProvider.of(context);
|
||||
|
||||
settings.updateFeed(
|
||||
feed.id,
|
||||
enabled: value
|
||||
);
|
||||
|
||||
loadPage();
|
||||
},
|
||||
),
|
||||
],
|
||||
),
|
||||
|
||||
SelectableText(
|
||||
feed.url,
|
||||
style: TextStyle(
|
||||
decoration: feed.enabled ? TextDecoration.none : TextDecoration.lineThrough
|
||||
),
|
||||
).muted.small,
|
||||
|
||||
Gap(12),
|
||||
],
|
||||
),
|
||||
|
||||
Gap(12),
|
||||
|
||||
Divider(),
|
||||
|
||||
Gap(16),
|
||||
|
||||
Text(
|
||||
"Application Data"
|
||||
).h4,
|
||||
|
||||
Gap(12),
|
||||
|
||||
Text(
|
||||
"Application Storage Location"
|
||||
).small.normal,
|
||||
|
||||
Gap(8),
|
||||
|
||||
Row(
|
||||
children: [
|
||||
Expanded(
|
||||
child: TextField(
|
||||
placeholder: Text(
|
||||
"Select a diretory using Browse"
|
||||
),
|
||||
initialValue: appStorageLocation,
|
||||
enabled: false,
|
||||
),
|
||||
),
|
||||
|
||||
Gap(8),
|
||||
|
||||
Button.outline(
|
||||
child: Text(
|
||||
"Browse"
|
||||
),
|
||||
onPressed: () {
|
||||
|
||||
FilePicker.platform.getDirectoryPath().then((selectedPath) {
|
||||
if (selectedPath != null) {
|
||||
SettingsProvider settings = SettingsProvider.of(context);
|
||||
|
||||
settings.setApplicationStorageLocation(selectedPath);
|
||||
|
||||
loadPage();
|
||||
|
||||
}
|
||||
});
|
||||
|
||||
},
|
||||
),
|
||||
|
||||
Gap(8),
|
||||
|
||||
IconButton.destructive(
|
||||
icon: Icon(
|
||||
LucideIcons.refreshCcw
|
||||
),
|
||||
onPressed: () async {
|
||||
SettingsProvider settings = SettingsProvider.of(context);
|
||||
|
||||
settings.setApplicationStorageLocation((await getApplicationDocumentsDirectory()).path);
|
||||
|
||||
loadPage();
|
||||
},
|
||||
)
|
||||
],
|
||||
),
|
||||
|
||||
],
|
||||
).withMargin(
|
||||
all: 10
|
||||
),
|
||||
),
|
||||
),
|
||||
).withPadding(all: 24),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const SizedBox(height: 16),
|
||||
class AddFeedDialog extends StatelessWidget {
|
||||
|
||||
SizedBox(
|
||||
width: double.infinity,
|
||||
child: Card(
|
||||
child: Column(
|
||||
crossAxisAlignment: CrossAxisAlignment.start,
|
||||
Feed? existingFeed;
|
||||
|
||||
AddFeedDialog({super.key, this.existingFeed});
|
||||
|
||||
InputKey feedTitleKey = InputKey("feed_title");
|
||||
InputKey feedUrlKey = InputKey("feed_url");
|
||||
|
||||
@override
|
||||
Widget build(BuildContext context) {
|
||||
return AlertDialog(
|
||||
title: Text(
|
||||
existingFeed == null ? "Add Feed" : "Edit Feed"
|
||||
),
|
||||
content: SizedBox(
|
||||
width: 400,
|
||||
child: Form(
|
||||
onSubmit: (context, values) {
|
||||
String title = values[feedTitleKey] ?? "";
|
||||
String url = values[feedUrlKey] ?? "";
|
||||
|
||||
SettingsProvider settings = SettingsProvider.of(context);
|
||||
|
||||
if (existingFeed != null) {
|
||||
settings.updateFeed(
|
||||
existingFeed!.id,
|
||||
title: title,
|
||||
url: url
|
||||
);
|
||||
} else {
|
||||
settings.addFeed(Feed(
|
||||
title: title,
|
||||
url: url,
|
||||
enabled: true
|
||||
));
|
||||
}
|
||||
|
||||
Navigator.of(context).pop();
|
||||
},
|
||||
child: Column(
|
||||
crossAxisAlignment: CrossAxisAlignment.start,
|
||||
children: [
|
||||
|
||||
Text(
|
||||
"Add a custom RSS feed to your news sources."
|
||||
).muted,
|
||||
|
||||
Gap(16),
|
||||
|
||||
FormField(
|
||||
key: feedTitleKey,
|
||||
label: Text(
|
||||
"Title"
|
||||
),
|
||||
validator: ConditionalValidator((value) {
|
||||
if (value is String) {
|
||||
return value.trim().isNotEmpty;
|
||||
}
|
||||
return false;
|
||||
}, message: "Title cannot be empty"),
|
||||
child: TextField(
|
||||
initialValue: existingFeed?.title,
|
||||
),
|
||||
),
|
||||
|
||||
Gap(12),
|
||||
|
||||
FormField(
|
||||
key: feedUrlKey,
|
||||
label: Text(
|
||||
"Resource URL"
|
||||
),
|
||||
validator: ConditionalValidator((value) {
|
||||
if (value is String) {
|
||||
return value.trim().isNotEmpty;
|
||||
}
|
||||
return false;
|
||||
}, message: "URL cannot be empty"),
|
||||
child: TextField(
|
||||
initialValue: existingFeed?.url,
|
||||
)
|
||||
),
|
||||
|
||||
Gap(24),
|
||||
|
||||
Text(
|
||||
"Only use valid RSS feed URLs. Preferably only use sites you trust, and specifically business and financial news sources."
|
||||
),
|
||||
|
||||
Gap(24),
|
||||
|
||||
Row(
|
||||
mainAxisAlignment: MainAxisAlignment.end,
|
||||
children: [
|
||||
|
||||
Text(
|
||||
"Feeds"
|
||||
).extraBold,
|
||||
|
||||
const SizedBox(height: 16),
|
||||
|
||||
TextField(
|
||||
|
||||
Button.outline(
|
||||
child: Text(
|
||||
"Cancel"
|
||||
),
|
||||
onPressed: () {
|
||||
Navigator.of(context).pop();
|
||||
},
|
||||
),
|
||||
|
||||
],
|
||||
),
|
||||
),
|
||||
)
|
||||
Gap(8),
|
||||
|
||||
],
|
||||
).withMargin(
|
||||
all: 10
|
||||
SubmitButton(
|
||||
child: Text(
|
||||
existingFeed != null ? "Update Feed" : "Add Feed"
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
],
|
||||
),
|
||||
),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user