Skip to content

Commit

Permalink
Speed up test setup (#118)
Browse files Browse the repository at this point in the history 
* configure separate test for provisioning using real API

* Add integration test for provisioning UI

* clean up logging
  • Loading branch information
@Barabas5532
Barabas5532 authored Jul 23, 2024
1 parent 3b568bb commit 3b74565
Show file tree
Hide file tree
Showing 10 changed files with 389 additions and 247 deletions.
270 changes: 157 additions & 113 deletions frontend/integration_test/simple_test.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,15 @@
@Tags(['api'])
library;

import 'package:flutter/material.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:logging/logging.dart';
import 'package:shrapnel/main.dart';
import 'package:shrapnel/midi_mapping/model/models.dart';

import '../test/firmware_api_test/util.dart';
import '../test/home_page_object.dart';
import '../test/util.dart';

// preset test
// start up
Expand Down Expand Up @@ -82,8 +84,7 @@ const networkSsid = String.fromEnvironment('NETWORK_SSID');
const networkPassphrase = String.fromEnvironment('NETWORK_PASSPHRASE');
// ignore: do_not_use_environment
const firmwareBinaryPath = String.fromEnvironment('FIRMWARE_BINARY_PATH');
// ignore: do_not_use_environment
const useFastProvisioning = bool.fromEnvironment('FAST_PROVISIONING');
const port = '/dev/ttyUSB0';

final _log = Logger('test');

Expand All @@ -96,136 +97,179 @@ void main() {

setUpAll(() async {
macAddress = await flashFirmware(
port: port,
appPartitionAddress: 0x10000,
path: firmwareBinaryPath,
);
});

setUp(() async {
await nvsErase();
testWidgets('wifi provisioning', (tester) async {
await nvsErase(port: port);

uart = await ShrapnelUart.open(port);
addTearDown(() => uart.dispose());

await connectToDutAccessPoint(macAddress);

// mDNS seems to be slow, use the IP address directly for faster testing.
await tester.pumpWidget(
App(
normalHost: dutIpAddress,
provisioningHost: '192.168.4.1',
),
);

final homePage = HomePageObject(tester);
final provisioningPage = await homePage.openWifiProvisioningPage();

await provisioningPage.startProvisioning();

await pumpWaitingFor(
tester: tester,
predicate: () =>
provisioningPage.findScanCompletePage.evaluate().isNotEmpty,
timeout: const Duration(seconds: 10),
);

await provisioningPage.openAdvancedSetup();

await provisioningPage.enterSsid(networkSsid);
await provisioningPage.enterPassword(networkPassphrase);

await provisioningPage.submitAdvanced();

await pumpWaitingFor(
tester: tester,
predicate: () => provisioningPage.findSuccessPage.evaluate().isNotEmpty,
timeout: const Duration(seconds: 10),
);

await tester.tap(find.byType(BackButton));

await pumpWaitingFor(
tester: tester,
predicate: () => homePage.findHomePage.evaluate().isNotEmpty,
timeout: const Duration(seconds: 10),
);

await homePage.waitUntilConnected();
});

group('wifi already set up', () {
setUp(() async {
await nvsErase(port: port);

uart = await ShrapnelUart.open('/dev/ttyUSB0');
// TODO move this into the uart class, so it logs by itself even if there
// are no external log listeners
// ensure at least one listener so logging side effect always runs
uart.log.listen((_) {});
addTearDown(uart.dispose);
uart = await ShrapnelUart.open(port);
addTearDown(uart.dispose);

if (useFastProvisioning) {
_log.warning('Bypassing Wi-Fi provisioning to speed up test execution');
await uart.provisionWifi(ssid: networkSsid, password: networkPassphrase);
} else {
// TODO we can still speed up tests where provisioning must be tested:
// - export NVS partition to file after first time wifi provisioning is used
// - reload it in the setup for each following test. This resets the NVS
// partition without requiring a fresh wifi provisioning run for each
// test.
//
// Alternatively, create a new test group that doesn't run the fast wifi
// provisioning setup, and instead runs the slow wifi setup.

await connectToDutAccessPoint(macAddress);
await setUpWiFi(ssid: networkSsid, password: networkPassphrase);
}
// Wait for firmware to start server after getting provisioned
await Future<void>.delayed(const Duration(seconds: 10));
});

// Wait for firmware to start server after getting provisioned
await Future<void>.delayed(const Duration(seconds: 10));
});
testWidgets('uart console smoke', (tester) async {
await tester.runAsync(() async {
// At least one response line has to arrive within 1 second
final responseLogs = uart.log.first.timeout(const Duration(seconds: 1));

await uart.sendMidiMessage(
const MidiMessage.controlChange(
channel: 0,
control: 1,
value: 0x00,
),
);

await expectLater(responseLogs, completes);
});
});

testWidgets('uart console smoke', (tester) async {
await tester.runAsync(() async {
// At least one response line has to arrive within 1 second
final responseLogs = uart.log.first.timeout(const Duration(seconds: 1));
testWidgets('audio parameters', (tester) async {
await tester.runAsync(() async {
// audio parameter basic test
// factory reset
// all parameters loaded correctly
// update a parameter
// force save to NVS
// reboot
// check parameter reloaded
});
});

await uart.sendMidiMessage(
const MidiMessage.controlChange(
channel: 0,
control: 1,
value: 0x00,
testWidgets('simple test', (tester) async {
// mDNS seems to be slow, use the IP address directly for faster testing.
await tester.pumpWidget(
App(
normalHost: dutIpAddress,
provisioningHost: '192.168.4.1',
),
);

await expectLater(responseLogs, completes);
});
});
// wait until connected
// poll connection status widget until ready with timeout
final homePage = HomePageObject(tester);
await homePage.waitUntilConnected();

testWidgets('audio parameters', (tester) async {
await tester.runAsync(() async {
// audio parameter basic test
// factory reset
// all parameters loaded correctly
// update a parameter
// force save to NVS
// reboot
// check parameter reloaded
});
});
await homePage.createPreset('Preset 1');

testWidgets('simple test', (tester) async {
await tester.pumpWidget(App());
final midiMappingPage = await homePage.openMidiMapping();

// wait until connected
// poll connection status widget until ready with timeout
final homePage = HomePageObject(tester);
await homePage.waitUntilConnected();
expect(midiMappingPage.findPage(), findsOneWidget);
expect(midiMappingPage.findMappingRows(), findsNothing);

await homePage.createPreset('Preset 1');

final midiMappingPage = await homePage.openMidiMapping();

expect(midiMappingPage.findPage(), findsOneWidget);
expect(midiMappingPage.findMappingRows(), findsNothing);

// create a mapping
final midiMappingCreatePage = await midiMappingPage.openCreateDialog();
await midiMappingCreatePage.selectMidiChannel(1);
await midiMappingCreatePage.selectCcNumber(2);
await midiMappingCreatePage.selectMode(MidiMappingMode.parameter);
// XXX: There is a bug in flutter where the DropdownButton's popup menu is
// unreliable during tests: https://github.com/flutter/flutter/issues/82908
//
// Pick an arbitrary parameter here. The only criteria for selection is that
// it actually works during the test. This is more likely if something is
// picked from the top of the list.
await midiMappingCreatePage.selectParameter('Chorus: DEPTH');
await midiMappingCreatePage.submitCreateDialog();

await tester.pump(const Duration(seconds: 1));
await tester.pumpAndSettle();

// Expect new mapping visible in UI
expect(midiMappingPage.findMappingRows(), findsOneWidget);

await midiMappingPage.openCreateDialog();
await midiMappingCreatePage.selectMidiChannel(2);
await midiMappingCreatePage.selectCcNumber(3);
await midiMappingCreatePage.selectMode(MidiMappingMode.button);
await midiMappingCreatePage.selectPreset('Preset 1');
await midiMappingCreatePage.submitCreateDialog();

await tester.pump(const Duration(seconds: 1));
await tester.pumpAndSettle();

// Expect new mapping visible in UI
expect(midiMappingPage.findMappingRows(), findsNWidgets(2));

await midiMappingPage.openCreateDialog();
await midiMappingCreatePage.selectMidiChannel(3);
await midiMappingCreatePage.selectCcNumber(4);
await midiMappingCreatePage.selectMode(MidiMappingMode.toggle);
// XXX: There is a bug in flutter where the DropdownButton's popup menu is
// unreliable during tests: https://github.com/flutter/flutter/issues/82908
//
// Pick an arbitrary parameter here. The only criteria for selection is that
// it actually works during the test. This is more likely if something is
// picked from the top of the list.
await midiMappingCreatePage.selectParameter('Chorus: RATE');
await midiMappingCreatePage.submitCreateDialog();

await tester.pump(const Duration(seconds: 1));
await tester.pumpAndSettle();

// Expect new mapping visible in UI
expect(midiMappingPage.findMappingRows(), findsNWidgets(3));
// create a mapping
final midiMappingCreatePage = await midiMappingPage.openCreateDialog();
await midiMappingCreatePage.selectMidiChannel(1);
await midiMappingCreatePage.selectCcNumber(2);
await midiMappingCreatePage.selectMode(MidiMappingMode.parameter);
// XXX: There is a bug in flutter where the DropdownButton's popup menu is
// unreliable during tests: https://github.com/flutter/flutter/issues/82908
//
// Pick an arbitrary parameter here. The only criteria for selection is that
// it actually works during the test. This is more likely if something is
// picked from the top of the list.
await midiMappingCreatePage.selectParameter('Chorus: DEPTH');
await midiMappingCreatePage.submitCreateDialog();

await tester.pump(const Duration(seconds: 1));
await tester.pumpAndSettle();

// Expect new mapping visible in UI
expect(midiMappingPage.findMappingRows(), findsOneWidget);

await midiMappingPage.openCreateDialog();
await midiMappingCreatePage.selectMidiChannel(2);
await midiMappingCreatePage.selectCcNumber(3);
await midiMappingCreatePage.selectMode(MidiMappingMode.button);
await midiMappingCreatePage.selectPreset('Preset 1');
await midiMappingCreatePage.submitCreateDialog();

await tester.pump(const Duration(seconds: 1));
await tester.pumpAndSettle();

// Expect new mapping visible in UI
expect(midiMappingPage.findMappingRows(), findsNWidgets(2));

await midiMappingPage.openCreateDialog();
await midiMappingCreatePage.selectMidiChannel(3);
await midiMappingCreatePage.selectCcNumber(4);
await midiMappingCreatePage.selectMode(MidiMappingMode.toggle);
// XXX: There is a bug in flutter where the DropdownButton's popup menu is
// unreliable during tests: https://github.com/flutter/flutter/issues/82908
//
// Pick an arbitrary parameter here. The only criteria for selection is that
// it actually works during the test. This is more likely if something is
// picked from the top of the list.
await midiMappingCreatePage.selectParameter('Chorus: RATE');
await midiMappingCreatePage.submitCreateDialog();

await tester.pump(const Duration(seconds: 1));
await tester.pumpAndSettle();

// Expect new mapping visible in UI
expect(midiMappingPage.findMappingRows(), findsNWidgets(3));
});
});
}
10 changes: 8 additions & 2 deletions frontend/lib/main.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -108,9 +108,15 @@ class App extends StatelessWidget {
PresetsRepositoryBase? presetsRepository,
ParameterService? parameterService,
SelectedPresetRepositoryBase? selectedPresetRepository,
String? normalHost,
String? provisioningHost,
}) {
provisioningHost ??= 'guitar-dsp.local';
normalHost ??= 'guitar-dsp.local';

websocket ??= RobustWebsocket(
uri: Uri.parse('http://guitar-dsp.local:8080/websocket'),
uri: Uri.parse('http://guitar-dsp.local:8080/websocket')
.replace(host: normalHost),
);
_websocket = websocket;
apiWebsocket ??= ApiWebsocket(websocket: websocket);
Expand All @@ -125,7 +131,7 @@ class App extends StatelessWidget {
_log.info('Creating provisioning connection');
return Provisioning(
security: Security1(pop: 'abcd1234'),
transport: TransportHTTP('guitar-dsp.local'),
transport: TransportHTTP(provisioningHost!),
);
},
);
Expand Down
2 changes: 1 addition & 1 deletion frontend/lib/robust_websocket.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ class RobustWebsocket extends ChangeNotifier
while (socket == null) {
try {
final client = HttpClient();
client.connectionTimeout = const Duration(seconds: 15);
client.connectionTimeout = const Duration(seconds: 5);
final request = await client.openUrl('GET', uri);

final nonce = <int>[];
Expand Down
1 change: 1 addition & 0 deletions frontend/lib/wifi_provisioning.dart
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,6 +309,7 @@ class _WifiScanningScreenState extends State<_WifiScanningScreen> {
child = Padding(
padding: const EdgeInsets.all(8),
child: Column(
key: const Key('wifi scan complete page'),
children: [
Expanded(
child: ListView.builder(
Expand Down
Loading

0 comments on commit 3b74565

Please sign in to comment.