Skip to content

Commit d218780

Browse files
devstressdevstress
authored
Merge pull request #354 from devstress/copilot/fix-failed-tests-and-update-readme
Fix LearningCourse integration test failures and add manual execution documentation
2 parents 4ac3198 + 07adc9f commit d218780

File tree

52 files changed

+1944
-104
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+1944
-104
lines changed

‎LearningCourse/Day01-Kafka-Flink-Data-Pipeline/Exercise-Solutions/Exercise1-StringCapitalize/Program.cs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -273,7 +273,7 @@ static async Task ConsumeResults()
273273
var partitionCounts = new Dictionary<int, int>();
274274
var allMessages = new List<(int partition, string value)>();
275275
var stopwatch = Stopwatch.StartNew();
276-
var timeout = TimeSpan.FromSeconds(30);
276+
var timeout = TimeSpan.FromSeconds(60);
277277

278278
try
279279
{
@@ -574,7 +574,7 @@ static async Task CreateTopicsAsync()
574574

575575
static async Task WaitForKafkaReadyAsync()
576576
{
577-
var timeout = TimeSpan.FromSeconds(30); // Increased from 20s to 30s - Confluent Local takes time to initialize
577+
var timeout = TimeSpan.FromSeconds(60); // Increased from 20s to 30s - Confluent Local takes time to initialize
578578
var stopwatch = Stopwatch.StartNew();
579579
var retryDelay = 1000; // Start with 1 second
580580

@@ -668,7 +668,7 @@ static async Task WaitForKafkaReadyAsync()
668668

669669
static async Task WaitForFlinkHealthyAsync()
670670
{
671-
var timeout = TimeSpan.FromSeconds(30);
671+
var timeout = TimeSpan.FromSeconds(60);
672672
var stopwatch = Stopwatch.StartNew();
673673
var retryDelay = 1000;
674674

‎LearningCourse/Day01-Kafka-Flink-Data-Pipeline/Exercise-Solutions/Exercise2-BackupAggregator/Program.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -573,7 +573,7 @@ static async Task CreateTopicsAsync()
573573

574574
static async Task WaitForKafkaReadyAsync()
575575
{
576-
var timeout = TimeSpan.FromSeconds(30); // Increased from 20s to 30s - Confluent Local takes time to initialize
576+
var timeout = TimeSpan.FromSeconds(60); // Increased from 20s to 30s - Confluent Local takes time to initialize
577577
var stopwatch = Stopwatch.StartNew();
578578
var retryDelay = 1000; // Start with 1 second
579579

@@ -671,7 +671,7 @@ static async Task WaitForKafkaReadyAsync()
671671
/// </summary>
672672
static async Task WaitForFlinkHealthyAsync()
673673
{
674-
var timeout = TimeSpan.FromSeconds(30);
674+
var timeout = TimeSpan.FromSeconds(60);
675675
var stopwatch = Stopwatch.StartNew();
676676
var retryDelay = 1000; // Start with 1 second
677677

‎LearningCourse/Day01-Kafka-Flink-Data-Pipeline/README.md‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -496,3 +496,111 @@ This executes all steps from the tutorial:
496496
- 📖 **Apache Flink**: [https://flink.apache.org/](https://flink.apache.org/)
497497
- 🔧 **Confluent Kafka .NET**: [https://docs.confluent.io/kafka-clients/dotnet/current/overview.html](https://docs.confluent.io/kafka-clients/dotnet/current/overview.html)
498498
- 💻 **FlinkDotNet**: Repository documentation in `docs/` folder
499+
500+
501+
## Running Exercises Manually
502+
503+
The exercises can be run manually outside of the integration tests. This requires starting the infrastructure and setting environment variables that are normally discovered automatically by the test framework.
504+
505+
### Step 1: Start Infrastructure
506+
507+
From the repository root, start the LocalTesting infrastructure in LearningCourse mode:
508+
509+
```bash
510+
# Linux/macOS
511+
cd LocalTesting
512+
./run-learningcourse.sh
513+
514+
# Windows (PowerShell)
515+
cd LocalTesting
516+
$env:LEARNINGCOURSE="true"
517+
dotnet run --project LocalTesting.FlinkSqlAppHost --configuration Release
518+
```
519+
520+
This starts:
521+
- Apache Flink cluster (JobManager + TaskManager + SQL Gateway)
522+
- Apache Kafka with JMX metrics
523+
- FlinkDotNet Gateway (port 8086)
524+
- Temporal workflow server (optional, for Day06+)
525+
- Redis (for state management)
526+
- Prometheus (metrics collection)
527+
- Grafana (metrics visualization)
528+
529+
Wait approximately 60 seconds for all containers to be ready.
530+
531+
### Step 2: Discover Service Endpoints
532+
533+
The infrastructure uses dynamic port allocation. You need to discover the actual ports assigned:
534+
535+
1. **Open Aspire Dashboard**: The AppHost will display a URL like `http://localhost:15000`
536+
2. **Find Kafka Port**: Look for "kafka" service, note the host port (e.g., `localhost:32785`)
537+
3. **Find Flink JobManager Port**: Look for "flink-jobmanager-jm-http" service, note the port (e.g., `localhost:32787`)
538+
539+
### Step 3: Set Environment Variables
540+
541+
Before running an exercise, set these environment variables:
542+
543+
```bash
544+
# Linux/macOS
545+
export KAFKA_BOOTSTRAP_SERVERS="localhost:XXXXX" # Replace XXXXX with discovered Kafka host port
546+
export KAFKA_FLINK_BOOTSTRAP_SERVERS="kafka:9093" # Fixed container-to-container address
547+
export FLINK_JOB_GATEWAY_URL="http://localhost:8086/" # Fixed JobGateway port
548+
export FLINK_JOBMANAGER_URL="http://localhost:YYYYY" # Replace YYYYY with discovered Flink port
549+
550+
# Windows (PowerShell)
551+
$env:KAFKA_BOOTSTRAP_SERVERS="localhost:XXXXX"
552+
$env:KAFKA_FLINK_BOOTSTRAP_SERVERS="kafka:9093"
553+
$env:FLINK_JOB_GATEWAY_URL="http://localhost:8086/"
554+
$env:FLINK_JOBMANAGER_URL="http://localhost:YYYYY"
555+
```
556+
557+
**Optional environment variables** (depending on the exercise):
558+
```bash
559+
# For Day06 Temporal exercises
560+
export TEMPORAL_ENDPOINT="localhost:ZZZZZ" # Replace with discovered Temporal port
561+
562+
# For exercises using Redis
563+
export REDIS_ENDPOINT="localhost:WWWWW" # Replace with discovered Redis port
564+
```
565+
566+
### Step 4: Run Exercise
567+
568+
Navigate to the exercise directory and run:
569+
570+
```bash
571+
cd Day01-Kafka-Flink-Data-Pipeline/Exercise-Solutions/Exercise1-StringCapitalize
572+
dotnet run --configuration Release
573+
```
574+
575+
### Environment Variable Reference
576+
577+
| Variable | Purpose | Example Value |
578+
|----------|---------|---------------|
579+
| `KAFKA_BOOTSTRAP_SERVERS` | Kafka address for producer/consumer on host | `localhost:32785` |
580+
| `KAFKA_FLINK_BOOTSTRAP_SERVERS` | Kafka address for Flink jobs (container-to-container) | `kafka:9093` |
581+
| `FLINK_JOB_GATEWAY_URL` | FlinkDotNet Gateway endpoint for job submission | `http://localhost:8086/` |
582+
| `FLINK_JOBMANAGER_URL` | Flink JobManager REST API for health checks | `http://localhost:32787` |
583+
| `TEMPORAL_ENDPOINT` | Temporal server endpoint (Day06+) | `localhost:32789` |
584+
| `REDIS_ENDPOINT` | Redis endpoint for state management | `localhost:32783` |
585+
586+
### Why Dynamic Ports?
587+
588+
The test infrastructure uses .NET Aspire which assigns dynamic ports to avoid conflicts. This is why you need to discover ports from the Aspire Dashboard rather than using hardcoded values.
589+
590+
### Alternative: Use Integration Tests
591+
592+
For automated testing with automatic port discovery, use the integration test framework:
593+
594+
```bash
595+
# Run all Day01 tests
596+
dotnet test LearningCourse/IntegrationTests.sln --filter "FullyQualifiedName~Day01Tests"
597+
```
598+
599+
The integration tests automatically:
600+
- Start the infrastructure
601+
- Discover service endpoints
602+
- Set environment variables
603+
- Run exercises
604+
- Validate results
605+
- Clean up resources
606+

‎LearningCourse/Day02-Flink21-Fundamentals/README.md‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1696,3 +1696,111 @@ You've successfully set up and validated a **production-grade streaming infrastr
16961696

16971697
**Next**: [Day 2: Real-World Stream Processing Patterns →](../Day02-Stream-Processing-Patterns/README.md)
16981698

1699+
1700+
1701+
## Running Exercises Manually
1702+
1703+
The exercises can be run manually outside of the integration tests. This requires starting the infrastructure and setting environment variables that are normally discovered automatically by the test framework.
1704+
1705+
### Step 1: Start Infrastructure
1706+
1707+
From the repository root, start the LocalTesting infrastructure in LearningCourse mode:
1708+
1709+
```bash
1710+
# Linux/macOS
1711+
cd LocalTesting
1712+
./run-learningcourse.sh
1713+
1714+
# Windows (PowerShell)
1715+
cd LocalTesting
1716+
$env:LEARNINGCOURSE="true"
1717+
dotnet run --project LocalTesting.FlinkSqlAppHost --configuration Release
1718+
```
1719+
1720+
This starts:
1721+
- Apache Flink cluster (JobManager + TaskManager + SQL Gateway)
1722+
- Apache Kafka with JMX metrics
1723+
- FlinkDotNet Gateway (port 8086)
1724+
- Temporal workflow server (optional, for Day06+)
1725+
- Redis (for state management)
1726+
- Prometheus (metrics collection)
1727+
- Grafana (metrics visualization)
1728+
1729+
Wait approximately 60 seconds for all containers to be ready.
1730+
1731+
### Step 2: Discover Service Endpoints
1732+
1733+
The infrastructure uses dynamic port allocation. You need to discover the actual ports assigned:
1734+
1735+
1. **Open Aspire Dashboard**: The AppHost will display a URL like `http://localhost:15000`
1736+
2. **Find Kafka Port**: Look for "kafka" service, note the host port (e.g., `localhost:32785`)
1737+
3. **Find Flink JobManager Port**: Look for "flink-jobmanager-jm-http" service, note the port (e.g., `localhost:32787`)
1738+
1739+
### Step 3: Set Environment Variables
1740+
1741+
Before running an exercise, set these environment variables:
1742+
1743+
```bash
1744+
# Linux/macOS
1745+
export KAFKA_BOOTSTRAP_SERVERS="localhost:XXXXX" # Replace XXXXX with discovered Kafka host port
1746+
export KAFKA_FLINK_BOOTSTRAP_SERVERS="kafka:9093" # Fixed container-to-container address
1747+
export FLINK_JOB_GATEWAY_URL="http://localhost:8086/" # Fixed JobGateway port
1748+
export FLINK_JOBMANAGER_URL="http://localhost:YYYYY" # Replace YYYYY with discovered Flink port
1749+
1750+
# Windows (PowerShell)
1751+
$env:KAFKA_BOOTSTRAP_SERVERS="localhost:XXXXX"
1752+
$env:KAFKA_FLINK_BOOTSTRAP_SERVERS="kafka:9093"
1753+
$env:FLINK_JOB_GATEWAY_URL="http://localhost:8086/"
1754+
$env:FLINK_JOBMANAGER_URL="http://localhost:YYYYY"
1755+
```
1756+
1757+
**Optional environment variables** (depending on the exercise):
1758+
```bash
1759+
# For Day06 Temporal exercises
1760+
export TEMPORAL_ENDPOINT="localhost:ZZZZZ" # Replace with discovered Temporal port
1761+
1762+
# For exercises using Redis
1763+
export REDIS_ENDPOINT="localhost:WWWWW" # Replace with discovered Redis port
1764+
```
1765+
1766+
### Step 4: Run Exercise
1767+
1768+
Navigate to the exercise directory and run:
1769+
1770+
```bash
1771+
cd Day01-Kafka-Flink-Data-Pipeline/Exercise-Solutions/Exercise1-StringCapitalize
1772+
dotnet run --configuration Release
1773+
```
1774+
1775+
### Environment Variable Reference
1776+
1777+
| Variable | Purpose | Example Value |
1778+
|----------|---------|---------------|
1779+
| `KAFKA_BOOTSTRAP_SERVERS` | Kafka address for producer/consumer on host | `localhost:32785` |
1780+
| `KAFKA_FLINK_BOOTSTRAP_SERVERS` | Kafka address for Flink jobs (container-to-container) | `kafka:9093` |
1781+
| `FLINK_JOB_GATEWAY_URL` | FlinkDotNet Gateway endpoint for job submission | `http://localhost:8086/` |
1782+
| `FLINK_JOBMANAGER_URL` | Flink JobManager REST API for health checks | `http://localhost:32787` |
1783+
| `TEMPORAL_ENDPOINT` | Temporal server endpoint (Day06+) | `localhost:32789` |
1784+
| `REDIS_ENDPOINT` | Redis endpoint for state management | `localhost:32783` |
1785+
1786+
### Why Dynamic Ports?
1787+
1788+
The test infrastructure uses .NET Aspire which assigns dynamic ports to avoid conflicts. This is why you need to discover ports from the Aspire Dashboard rather than using hardcoded values.
1789+
1790+
### Alternative: Use Integration Tests
1791+
1792+
For automated testing with automatic port discovery, use the integration test framework:
1793+
1794+
```bash
1795+
# Run all Day01 tests
1796+
dotnet test LearningCourse/IntegrationTests.sln --filter "FullyQualifiedName~Day01Tests"
1797+
```
1798+
1799+
The integration tests automatically:
1800+
- Start the infrastructure
1801+
- Discover service endpoints
1802+
- Set environment variables
1803+
- Run exercises
1804+
- Validate results
1805+
- Clean up resources
1806+

‎LearningCourse/Day03-AI-Stream-Processing/Exercise-Solutions/Exercise31/Program.cs‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,9 @@ class Program
2727

2828
private static string FlinkGatewayUrl =>
2929
Environment.GetEnvironmentVariable("FLINK_JOB_GATEWAY_URL") ?? "http://localhost:8086";
30+
31+
private static string FlinkJobManagerUrl =>
32+
Environment.GetEnvironmentVariable("FLINK_JOBMANAGER_URL") ?? "http://localhost:8081";
3033

3134
private const string RegistrationTopic = "ai-model-registrations";
3235
private const string ValidationTopic = "ai-model-validations";
@@ -326,7 +329,7 @@ private static async Task CreateTopicsAsync()
326329

327330
private static async Task WaitForKafkaReadyAsync()
328331
{
329-
var timeout = TimeSpan.FromSeconds(30);
332+
var timeout = TimeSpan.FromSeconds(60);
330333
var stopwatch = Stopwatch.StartNew();
331334

332335
while (stopwatch.Elapsed < timeout)
@@ -361,15 +364,15 @@ private static async Task WaitForKafkaReadyAsync()
361364

362365
private static async Task WaitForFlinkHealthyAsync()
363366
{
364-
var timeout = TimeSpan.FromSeconds(30);
367+
var timeout = TimeSpan.FromSeconds(60);
365368
var stopwatch = Stopwatch.StartNew();
366369

367370
while (stopwatch.Elapsed < timeout)
368371
{
369372
try
370373
{
371374
using var httpClient = new System.Net.Http.HttpClient { Timeout = TimeSpan.FromSeconds(2) };
372-
var response = await httpClient.GetAsync($"{FlinkGatewayUrl}/api/v1/health");
375+
var response = await httpClient.GetAsync($"{FlinkJobManagerUrl}/v1/overview");
373376

374377
if (response.IsSuccessStatusCode)
375378
{

‎LearningCourse/Day03-AI-Stream-Processing/Exercise-Solutions/Exercise32/Program.cs‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ public class Program
3030

3131
private static string FlinkGatewayUrl =>
3232
Environment.GetEnvironmentVariable("FLINK_JOB_GATEWAY_URL") ?? "http://localhost:8086";
33+
34+
private static string FlinkJobManagerUrl =>
35+
Environment.GetEnvironmentVariable("FLINK_JOBMANAGER_URL") ?? "http://localhost:8081";
3336

3437
public static async Task Main(string[] args)
3538
{
@@ -331,7 +334,7 @@ static async Task ConsumeFraudAlerts()
331334
var consumedCount = 0;
332335
var highRiskCount = 0;
333336
var stopwatch = Stopwatch.StartNew();
334-
var timeout = TimeSpan.FromSeconds(30);
337+
var timeout = TimeSpan.FromSeconds(60);
335338

336339
try
337340
{
@@ -438,7 +441,7 @@ static async Task CreateTopicsAsync()
438441

439442
static async Task WaitForKafkaReadyAsync()
440443
{
441-
var timeout = TimeSpan.FromSeconds(30);
444+
var timeout = TimeSpan.FromSeconds(60);
442445
var stopwatch = Stopwatch.StartNew();
443446
var retryDelay = 1000;
444447

@@ -478,7 +481,7 @@ static async Task WaitForKafkaReadyAsync()
478481

479482
static async Task WaitForFlinkHealthyAsync()
480483
{
481-
var timeout = TimeSpan.FromSeconds(30);
484+
var timeout = TimeSpan.FromSeconds(60);
482485
var stopwatch = Stopwatch.StartNew();
483486
var retryDelay = 1000;
484487

@@ -487,7 +490,7 @@ static async Task WaitForFlinkHealthyAsync()
487490
try
488491
{
489492
using var httpClient = new System.Net.Http.HttpClient { Timeout = TimeSpan.FromSeconds(2) };
490-
var response = await httpClient.GetAsync($"{FlinkGatewayUrl}/api/v1/health");
493+
var response = await httpClient.GetAsync($"{FlinkJobManagerUrl}/v1/overview");
491494

492495
if (response.IsSuccessStatusCode)
493496
{

‎LearningCourse/Day03-AI-Stream-Processing/Exercise-Solutions/Exercise33/Program.cs‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@ class Program
3434

3535
private static string FlinkGatewayUrl =>
3636
Environment.GetEnvironmentVariable("FLINK_JOB_GATEWAY_URL") ?? "http://localhost:8086";
37+
38+
private static string FlinkJobManagerUrl =>
39+
Environment.GetEnvironmentVariable("FLINK_JOBMANAGER_URL") ?? "http://localhost:8081";
3740

3841
private const string InputTopic = "fraud-transactions-input";
3942
private const string PredictionsTopic = "fraud-model-predictions";
@@ -433,7 +436,7 @@ private static async Task CreateTopicsAsync()
433436

434437
private static async Task WaitForKafkaReadyAsync()
435438
{
436-
var timeout = TimeSpan.FromSeconds(30);
439+
var timeout = TimeSpan.FromSeconds(60);
437440
var stopwatch = Stopwatch.StartNew();
438441

439442
while (stopwatch.Elapsed < timeout)
@@ -468,15 +471,15 @@ private static async Task WaitForKafkaReadyAsync()
468471

469472
private static async Task WaitForFlinkHealthyAsync()
470473
{
471-
var timeout = TimeSpan.FromSeconds(30);
474+
var timeout = TimeSpan.FromSeconds(60);
472475
var stopwatch = Stopwatch.StartNew();
473476

474477
while (stopwatch.Elapsed < timeout)
475478
{
476479
try
477480
{
478481
using var httpClient = new System.Net.Http.HttpClient { Timeout = TimeSpan.FromSeconds(2) };
479-
var response = await httpClient.GetAsync($"{FlinkGatewayUrl}/api/v1/health");
482+
var response = await httpClient.GetAsync($"{FlinkJobManagerUrl}/v1/overview");
480483

481484
if (response.IsSuccessStatusCode)
482485
{

0 commit comments

Comments
 (0)