feature #834 Enabling files upload (Lustmored, weaverryan)

This PR was merged into the 2.x branch.

Discussion
----------

Enabling files upload

| Q             | A
| ------------- | ---
| Bug fix?      | no
| New feature?  | yes
| Tickets       |
| License       | MIT

Getting back on the file uploading front I've decided to go slowly this time, as in the end it should allow us to get basic support faster and then build up on it, without determining all the  little details of both backend and frontend implementations upfront.

This PR aims at *enabling* frontend tools to send files back to the server. It does not add file upload functions to the live controller or handle the files on the backend. Instead it focuses only on changes that are essential to get any kind of file upload in the future.

What exactly is done here:
- `RequestBuilder` gets an additional parameter (`files`) that takes a map of `FileList` objects that should be uploaded with the request (and the keys by which they should be available). This additional parameter is added as an empty map everywhere right now.
- To make file upload possible the content type of a request is moved from JSON to the default (form type) and all the component related data is sent under `data` key.
- `RequestBuilder` tests are adapted to the changes and tests for appending files to the request are added.
- On the backend site `LiveComponentSubscriber` is adapted to read data from JSON-encoded `data` parameter instead of request content (in case of a POST request), without any further functional changes.
- All tests are adapted to send data in the new format.

Two tests are failing for me locally (and in 2 seemingly random CI configurations), while logic tells me they shouldn't and I'm unable to find the cause right now. Help would be appreciated.

I would love to get that first stepping stone finalized and merged before further work. It is a simple change, but has strong implications, therefore getting those out of the way will make further work much more comfortable. Also - splitting the work will hopefully make it easier for you to review.

Commits
-------

d29df00 Fixing test
380b492 phpcs
cc41980 [Upload] Updating demo
1a147a2 Adapt tests again
65e2794 Rewire files handling to actually send files only once and clear input afterwards
1f74e6d Clear pendingFiles after being sent
0e1013e Pump up docs
2155865 Stop using data-model on file inputs as an alternative. It's pointless
c74f689 Rebuild dist assets
c7c56f0 Use `expectException` to mitigate test failures.
cf8ef71 Code style
610b216 Add very simple upload files docs
7ea3fbb Add extremely simple upload files example
ad70f81 Hook up files in Component and build assets
458ba60 Add possibility to send files with an action
7fd6e5a Rebuild assets
fe69154 Read POST payload from `data` key instead of a request body
b5ff362 Add tests for passing files to RequestBuilder and make them pass
14a640b Make files mandatory in Backend.ts and pass empty map from controller
2d13c27 Allow files to be passed to `RequestBuilder` and move request body under 'data' key

Author: weaverryan

src/LiveComponent/assets/dist/Backend/Backend.d.ts

@@ -10,6 +10,8 @@ export interface BackendInterface {
         [key: string]: any;
     }, children: ChildrenFingerprints, updatedPropsFromParent: {
         [key: string]: any;
+    }, files: {
+        [key: string]: FileList;
     }): BackendRequest;
 }
 export interface BackendAction {
@@ -23,5 +25,7 @@ export default class implements BackendInterface {
         [key: string]: any;
     }, children: ChildrenFingerprints, updatedPropsFromParent: {
         [key: string]: any;
+    }, files: {
+        [key: string]: FileList;
     }): BackendRequest;
 }

src/LiveComponent/assets/dist/Backend/RequestBuilder.d.ts

@@ -7,6 +7,8 @@ export default class {
         [key: string]: any;
     }, children: ChildrenFingerprints, updatedPropsFromParent: {
         [key: string]: any;
+    }, files: {
+        [key: string]: FileList;
     }): {
         url: string;
         fetchOptions: RequestInit;

src/LiveComponent/assets/dist/Component/index.d.ts

@@ -20,6 +20,7 @@ export default class Component {
     defaultDebounce: number;
     private backendRequest;
     private pendingActions;
+    private pendingFiles;
     private isRequestPending;
     private requestDebounceTimeout;
     private nextRequestPromise;
@@ -40,6 +41,7 @@ export default class Component {
     set(model: string, value: any, reRender?: boolean, debounce?: number | boolean): Promise<BackendResponse>;
     getData(model: string): any;
     action(name: string, args?: any, debounce?: number | boolean): Promise<BackendResponse>;
+    files(key: string, input: HTMLInputElement): void;
     render(): Promise<BackendResponse>;
     getUnsyncedModels(): string[];
     addChild(child: Component, modelBindings?: ModelBinding[]): void;

src/LiveComponent/assets/dist/live_controller.d.ts

@@ -48,6 +48,7 @@ export default class LiveControllerDefault extends Controller<HTMLElement> imple
     component: Component;
     pendingActionTriggerModelElement: HTMLElement | null;
     private elementEventListeners;
+    private pendingFiles;
     static componentRegistry: ComponentRegistry;
     initialize(): void;
     connect(): void;

src/LiveComponent/assets/dist/live_controller.js

@@ -1724,6 +1724,7 @@ class Component {
         this.defaultDebounce = 150;
         this.backendRequest = null;
         this.pendingActions = [];
+        this.pendingFiles = {};
         this.isRequestPending = false;
         this.requestDebounceTimeout = null;
         this.children = new Map();
@@ -1801,6 +1802,9 @@ class Component {
         this.debouncedStartRequest(debounce);
         return promise;
     }
+    files(key, input) {
+        this.pendingFiles[key] = input;
+    }
     render() {
         const promise = this.nextRequestPromise;
         this.tryStartingRequest();
@@ -1900,7 +1904,13 @@ class Component {
         const thisPromiseResolve = this.nextRequestPromiseResolve;
         this.resetPromise();
         this.unsyncedInputsTracker.resetUnsyncedFields();
-        this.backendRequest = this.backend.makeRequest(this.valueStore.getOriginalProps(), this.pendingActions, this.valueStore.getDirtyProps(), this.getChildrenFingerprints(), this.valueStore.getUpdatedPropsFromParent());
+        const filesToSend = {};
+        for (const [key, value] of Object.entries(this.pendingFiles)) {
+            if (value.files) {
+                filesToSend[key] = value.files;
+            }
+        }
+        this.backendRequest = this.backend.makeRequest(this.valueStore.getOriginalProps(), this.pendingActions, this.valueStore.getDirtyProps(), this.getChildrenFingerprints(), this.valueStore.getUpdatedPropsFromParent(), filesToSend);
         this.hooks.triggerHook('loading.state:started', this.element, this.backendRequest);
         this.pendingActions = [];
         this.valueStore.flushDirtyPropsToPending();
@@ -1909,6 +1919,9 @@ class Component {
             this.backendRequest = null;
             const backendResponse = new BackendResponse(response);
             const html = await backendResponse.getBody();
+            for (const input of Object.values(this.pendingFiles)) {
+                input.value = '';
+            }
             const headers = backendResponse.response.headers;
             if (headers.get('Content-Type') !== 'application/vnd.live-component+html' && !headers.get('X-Live-Redirect')) {
                 const controls = { displayError: true };
@@ -2130,7 +2143,7 @@ class RequestBuilder {
         this.url = url;
         this.csrfToken = csrfToken;
     }
-    buildRequest(props, actions, updated, children, updatedPropsFromParent) {
+    buildRequest(props, actions, updated, children, updatedPropsFromParent, files) {
         const splitUrl = this.url.split('?');
         let [url] = splitUrl;
         const [, queryString] = splitUrl;
@@ -2139,8 +2152,10 @@ class RequestBuilder {
         fetchOptions.headers = {
             Accept: 'application/vnd.live-component+html',
         };
+        const totalFiles = Object.entries(files).reduce((total, current) => total + current.length, 0);
         const hasFingerprints = Object.keys(children).length > 0;
         if (actions.length === 0 &&
+            totalFiles === 0 &&
             this.willDataFitInUrl(JSON.stringify(props), JSON.stringify(updated), params, JSON.stringify(children), JSON.stringify(updatedPropsFromParent))) {
             params.set('props', JSON.stringify(props));
             params.set('updated', JSON.stringify(updated));
@@ -2154,18 +2169,18 @@ class RequestBuilder {
         }
         else {
             fetchOptions.method = 'POST';
-            fetchOptions.headers['Content-Type'] = 'application/json';
             const requestData = { props, updated };
             if (Object.keys(updatedPropsFromParent).length > 0) {
                 requestData.propsFromParent = updatedPropsFromParent;
             }
             if (hasFingerprints) {
                 requestData.children = children;
             }
+            if (this.csrfToken &&
+                (actions.length || totalFiles)) {
+                fetchOptions.headers['X-CSRF-TOKEN'] = this.csrfToken;
+            }
             if (actions.length > 0) {
-                if (this.csrfToken) {
-                    fetchOptions.headers['X-CSRF-TOKEN'] = this.csrfToken;
-                }
                 if (actions.length === 1) {
                     requestData.args = actions[0].args;
                     url += `/${encodeURIComponent(actions[0].name)}`;
@@ -2175,7 +2190,15 @@ class RequestBuilder {
                     requestData.actions = actions;
                 }
             }
-            fetchOptions.body = JSON.stringify(requestData);
+            const formData = new FormData();
+            formData.append('data', JSON.stringify(requestData));
+            for (const [key, value] of Object.entries(files)) {
+                const length = value.length;
+                for (let i = 0; i < length; ++i) {
+                    formData.append(key, value[i]);
+                }
+            }
+            fetchOptions.body = formData;
         }
         const paramsString = params.toString();
         return {
@@ -2193,8 +2216,8 @@ class Backend {
     constructor(url, csrfToken = null) {
         this.requestBuilder = new RequestBuilder(url, csrfToken);
     }
-    makeRequest(props, actions, updated, children, updatedPropsFromParent) {
-        const { url, fetchOptions } = this.requestBuilder.buildRequest(props, actions, updated, children, updatedPropsFromParent);
+    makeRequest(props, actions, updated, children, updatedPropsFromParent, files) {
+        const { url, fetchOptions } = this.requestBuilder.buildRequest(props, actions, updated, children, updatedPropsFromParent, files);
         return new BackendRequest(fetch(url, fetchOptions), actions.map((backendAction) => backendAction.name), Object.keys(updated));
     }
 }
@@ -2658,6 +2681,7 @@ class LiveControllerDefault extends Controller {
             { event: 'change', callback: (event) => this.handleChangeEvent(event) },
             { event: 'live:connect', callback: (event) => this.handleConnectedControllerEvent(event) },
         ];
+        this.pendingFiles = {};
     }
     initialize() {
         this.handleDisconnectedChildControllerEvent = this.handleDisconnectedChildControllerEvent.bind(this);
@@ -2706,6 +2730,7 @@ class LiveControllerDefault extends Controller {
         const directives = parseDirectives(rawAction);
         let debounce = false;
         directives.forEach((directive) => {
+            let pendingFiles = {};
             const validModifiers = new Map();
             validModifiers.set('prevent', () => {
                 event.preventDefault();
@@ -2721,6 +2746,14 @@ class LiveControllerDefault extends Controller {
             validModifiers.set('debounce', (modifier) => {
                 debounce = modifier.value ? parseInt(modifier.value) : true;
             });
+            validModifiers.set('files', (modifier) => {
+                if (!modifier.value) {
+                    pendingFiles = this.pendingFiles;
+                }
+                else if (this.pendingFiles[modifier.value]) {
+                    pendingFiles[modifier.value] = this.pendingFiles[modifier.value];
+                }
+            });
             directive.modifiers.forEach((modifier) => {
                 var _a;
                 if (validModifiers.has(modifier.name)) {
@@ -2730,6 +2763,12 @@ class LiveControllerDefault extends Controller {
                 }
                 console.warn(`Unknown modifier ${modifier.name} in action "${rawAction}". Available modifiers are: ${Array.from(validModifiers.keys()).join(', ')}.`);
             });
+            for (const [key, input] of Object.entries(pendingFiles)) {
+                if (input.files) {
+                    this.component.files(key, input);
+                }
+                delete this.pendingFiles[key];
+            }
             this.component.action(directive.action, directive.named, debounce);
             if (getModelDirectiveFromElement(event.currentTarget, false)) {
                 this.pendingActionTriggerModelElement = event.currentTarget;
@@ -2799,12 +2838,22 @@ class LiveControllerDefault extends Controller {
         this.updateModelFromElementEvent(target, 'change');
     }
     updateModelFromElementEvent(element, eventName) {
+        var _a;
         if (!elementBelongsToThisComponent(element, this.component)) {
             return;
         }
         if (!(element instanceof HTMLElement)) {
             throw new Error('Could not update model for non HTMLElement');
         }
+        if (element instanceof HTMLInputElement && element.type === 'file') {
+            const key = element.name;
+            if ((_a = element.files) === null || _a === void 0 ? void 0 : _a.length) {
+                this.pendingFiles[key] = element;
+            }
+            else if (this.pendingFiles[key]) {
+                delete this.pendingFiles[key];
+            }
+        }
         const modelDirective = getModelDirectiveFromElement(element, false);
         if (!modelDirective) {
             return;

src/LiveComponent/assets/src/Backend/Backend.ts

@@ -13,6 +13,7 @@ export interface BackendInterface {
         updated: {[key: string]: any},
         children: ChildrenFingerprints,
         updatedPropsFromParent: {[key: string]: any},
+        files: {[key: string]: FileList},
     ): BackendRequest;
 }
 
@@ -34,13 +35,15 @@ export default class implements BackendInterface {
         updated: {[key: string]: any},
         children: ChildrenFingerprints,
         updatedPropsFromParent: {[key: string]: any},
+        files: {[key: string]: FileList},
     ): BackendRequest {
         const { url, fetchOptions } = this.requestBuilder.buildRequest(
             props,
             actions,
             updated,
             children,
-            updatedPropsFromParent
+            updatedPropsFromParent,
+            files
         );
 
         return new BackendRequest(

src/LiveComponent/assets/src/Backend/RequestBuilder.ts

@@ -15,6 +15,7 @@ export default class {
         updated: {[key: string]: any},
         children: ChildrenFingerprints,
         updatedPropsFromParent: {[key: string]: any},
+        files: {[key: string]: FileList},
     ): { url: string; fetchOptions: RequestInit } {
         const splitUrl = this.url.split('?');
         let [url] = splitUrl;
@@ -26,9 +27,15 @@ export default class {
             Accept: 'application/vnd.live-component+html',
         };
 
+        const totalFiles = Object.entries(files).reduce(
+            (total, current) => total + current.length,
+            0
+        );
+
         const hasFingerprints = Object.keys(children).length > 0;
         if (
             actions.length === 0 &&
+            totalFiles === 0 &&
             this.willDataFitInUrl(JSON.stringify(props), JSON.stringify(updated), params, JSON.stringify(children), JSON.stringify(updatedPropsFromParent))
         ) {
             params.set('props', JSON.stringify(props));
@@ -42,7 +49,6 @@ export default class {
             fetchOptions.method = 'GET';
         } else {
             fetchOptions.method = 'POST';
-            fetchOptions.headers['Content-Type'] = 'application/json';
             const requestData: any = { props, updated };
             if (Object.keys(updatedPropsFromParent).length > 0) {
                 requestData.propsFromParent = updatedPropsFromParent;
@@ -51,11 +57,15 @@ export default class {
                 requestData.children = children;
             }
 
+            if (
+                this.csrfToken &&
+                (actions.length || totalFiles)
+            ) {
+                fetchOptions.headers['X-CSRF-TOKEN'] = this.csrfToken;
+            }
+
             if (actions.length > 0) {
                 // one or more ACTIONs
-                if (this.csrfToken) {
-                    fetchOptions.headers['X-CSRF-TOKEN'] = this.csrfToken;
-                }
 
                 if (actions.length === 1) {
                     // simple, single action
@@ -68,7 +78,17 @@ export default class {
                 }
             }
 
-            fetchOptions.body = JSON.stringify(requestData);
+            const formData = new FormData();
+            formData.append('data', JSON.stringify(requestData));
+
+            for(const [key, value] of Object.entries(files)) {
+                const length = value.length;
+                for (let i = 0; i < length ; ++i) {
+                    formData.append(key, value[i]);
+                }
+            }
+
+            fetchOptions.body = formData;
         }
 
         const paramsString = params.toString();

src/LiveComponent/assets/src/Component/index.ts

@@ -55,6 +55,8 @@ export default class Component {
     private backendRequest: BackendRequest|null = null;
     /** Actions that are waiting to be executed */
     private pendingActions: BackendAction[] = [];
+    /** Files that are waiting to be sent */
+    private pendingFiles: {[key: string]: HTMLInputElement} = {};
     /** Is a request waiting to be made? */
     private isRequestPending = false;
     /** Current "timeout" before the pending request should be sent. */
@@ -194,6 +196,10 @@ export default class Component {
         return promise;
     }
 
+    files(key: string, input: HTMLInputElement): void {
+        this.pendingFiles[key] = input;
+    }
+
     render(): Promise<BackendResponse> {
         const promise = this.nextRequestPromise;
         this.tryStartingRequest();
@@ -352,12 +358,20 @@ export default class Component {
         // they are now "in sync" (with some exceptions noted inside)
         this.unsyncedInputsTracker.resetUnsyncedFields();
 
+        const filesToSend: {[key: string]: FileList} = {};
+        for(const [key, value] of Object.entries(this.pendingFiles)) {
+            if (value.files) {
+                filesToSend[key] = value.files;
+            }
+        }
+
         this.backendRequest = this.backend.makeRequest(
             this.valueStore.getOriginalProps(),
             this.pendingActions,
             this.valueStore.getDirtyProps(),
             this.getChildrenFingerprints(),
             this.valueStore.getUpdatedPropsFromParent(),
+            filesToSend,
         );
         this.hooks.triggerHook('loading.state:started', this.element, this.backendRequest);
 
@@ -370,6 +384,11 @@ export default class Component {
             const backendResponse = new BackendResponse(response);
             const html = await backendResponse.getBody();
 
+            // clear sent files inputs
+            for(const input of Object.values(this.pendingFiles)) {
+                input.value = '';
+            }
+
             // if the response does not contain a component, render as an error
             const headers = backendResponse.response.headers;
             if (headers.get('Content-Type') !== 'application/vnd.live-component+html' && !headers.get('X-Live-Redirect')) {