diff --git a/src/libpriv/rpmostree-postprocess.c b/src/libpriv/rpmostree-postprocess.c
index f4a1d789..4d872cd6 100644
--- a/src/libpriv/rpmostree-postprocess.c
+++ b/src/libpriv/rpmostree-postprocess.c
@@ -1443,7 +1443,9 @@ rpmostree_treefile_postprocessing (int            rootfs_fd,
       /* Clone all the things */
 
       /* Note we need to make binpath *not* absolute here */
-      if (!glnx_file_copy_at (AT_FDCWD, src, NULL, rootfs_fd, binpath + 1,
+      const char *target_binpath = binpath + 1;
+      g_assert_cmpint (*target_binpath, !=, '/');
+      if (!glnx_file_copy_at (AT_FDCWD, src, NULL, rootfs_fd, target_binpath,
                               GLNX_FILE_COPY_NOXATTRS, cancellable, error))
         return FALSE;
 
@@ -1455,6 +1457,9 @@ rpmostree_treefile_postprocessing (int            rootfs_fd,
           return glnx_prefix_error (error, "While executing postprocessing script '%s'", bn);
       }
 
+      if (unlinkat (rootfs_fd, target_binpath, 0) < 0)
+        return glnx_throw_errno_prefix (error, "unlinkat(%s)", target_binpath);
+
       g_print ("Finished postprocessing script '%s'\n", bn);
     }
 
diff --git a/tests/compose-tests/test-install-langs.sh b/tests/compose-tests/test-install-langs.sh
index 6a82f920..d2f74106 100755
--- a/tests/compose-tests/test-install-langs.sh
+++ b/tests/compose-tests/test-install-langs.sh
@@ -30,4 +30,9 @@ assert_file_has_content out.txt 'Thu Jan  1 00:00:00 UTC 1970'
 ostree --repo=${repobuild} cat ${treeref} /usr/etc/lang-test.de.touch.txt > out.txt
 assert_file_has_content out.txt 'missing file operand'
 
+if ostree --repo=${repobuild} ls ${treeref} /usr/bin/rpmostree-postprocess-lang-test.sh 2>err.txt; then
+    assert_not_reached "we failed to unlink?"
+fi
+assert_file_has_content err.txt "error: No such file or directory"
+
 echo "ok install-langs"