Return EISDIR when trying to create a path ending in / with open (#23135)

If we open a path that in a / and we're about to create a file, instead
return EISDIR.

Author: hoodmane

src/library_fs.js

@@ -1060,9 +1060,11 @@ FS.staticInit();
         mode = 0;
       }
       var node;
+      var isDirPath;
       if (typeof path == 'object') {
         node = path;
       } else {
+        isDirPath = path.endsWith("/");
         // noent_okay makes it so that if the final component of the path
         // doesn't exist, lookupPath returns `node: undefined`. `path` will be
         // updated to point to the target of all symlinks.
@@ -1081,6 +1083,8 @@ FS.staticInit();
           if ((flags & {{{ cDefs.O_EXCL }}})) {
             throw new FS.ErrnoError({{{ cDefs.EEXIST }}});
           }
+        } else if (isDirPath) {
+          throw new FS.ErrnoError({{{ cDefs.EISDIR }}});
         } else {
           // node doesn't exist, try to create it
           node = FS.mknod(path, mode, 0);

test/fs/test_enotdir.c

@@ -0,0 +1,24 @@
+#include <errno.h>
+#include <fcntl.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <sys/stat.h>
+#include <assert.h>
+
+int main() {
+  {
+    int src_fd = open("file", O_CREAT | O_WRONLY, 0777);
+    assert(src_fd >= 0);
+    assert(close(src_fd) == 0);
+  }
+  {
+    assert(mkdir("file/blah", 0777) == -1);
+    assert(errno == ENOTDIR);
+  }
+  {
+    assert(open("./does-not-exist/", O_CREAT) == -1);
+    assert(errno == EISDIR);
+  }
+  printf("success\n");
+}

test/fs/test_enotdir.out

@@ -5756,9 +5756,11 @@ def test_fs_emptyPath(self):
 
   @no_windows('https://github.com/emscripten-core/emscripten/issues/8882')
   @crossplatform
-  @also_with_noderawfs
+  @also_with_nodefs_both
   def test_fs_enotdir(self):
-    self.do_run_in_out_file_test('fs/test_enotdir.c')
+    if MACOS and '-DNODERAWFS' in self.emcc_args:
+      self.skipTest('BSD libc sets a different errno')
+    self.do_runf('fs/test_fs_enotdir.c', 'success')
 
   @also_with_noderawfs
   def test_fs_append(self):