diff --git a/Makefile.am b/Makefile.am
index 9804b42..f7066b4 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -52,7 +52,10 @@ dist-hook: dist-check-valac
 distcleancheck_listfiles =  \
        find . -name '*.gc[dn][oa]' -prune -o -type f -print
 
-TESTS_ENVIRONMENT = LD_LIBRARY_PATH=$(builddir)/.libs GI_TYPELIB_PATH=$(builddir)
+TESTS_ENVIRONMENT = \
+	LD_LIBRARY_PATH=$(builddir)/.libs \
+	GI_TYPELIB_PATH=$(builddir) \
+	abs_top_builddir=$(abs_top_builddir)
 TEST_EXTENSIONS = .py .js
 
 # Default executable tests
diff --git a/meson.build b/meson.build
index f20a66e..bb58f68 100644
--- a/meson.build
+++ b/meson.build
@@ -70,6 +70,10 @@ conf.set('_DEBUG', enable_debug)
 conf.set('HAVE_MLOCK', meson.get_compiler('c').has_function('mlock'))
 configure_file(output: 'config.h', configuration: conf)
 
+# Test environment
+test_env = environment()
+test_env.set('abs_top_builddir', meson.build_root())
+
 # Subfolders
 subdir('po')
 subdir('egg')
diff --git a/tool/Makefile.am b/tool/Makefile.am
index 2361311..9d25ea1 100644
--- a/tool/Makefile.am
+++ b/tool/Makefile.am
@@ -5,3 +5,7 @@ tool_secret_tool_SOURCES = \
 
 tool_secret_tool_LDADD = \
 	libsecret-@SECRET_MAJOR@.la
+
+if WITH_GCRYPT
+TESTS += tool/test-secret-tool.sh
+endif
diff --git a/tool/meson.build b/tool/meson.build
index 686cf24..1bf9a84 100644
--- a/tool/meson.build
+++ b/tool/meson.build
@@ -9,3 +9,9 @@ secret_tool = executable('secret-tool',
   c_args: libsecret_cflags,
   install: true,
 )
+
+if with_gcrypt and host_machine.system() != 'windows'
+  test('test-secret-tool.sh',
+       find_program('test-secret-tool.sh'),
+       env: test_env)
+endif
diff --git a/tool/test-secret-tool.sh b/tool/test-secret-tool.sh
new file mode 100755
index 0000000..9bd4fbd
--- /dev/null
+++ b/tool/test-secret-tool.sh
@@ -0,0 +1,104 @@
+#!/bin/sh
+
+set -e
+
+testdir=$PWD/test-secret-tool-$$
+test -d "$testdir" || mkdir "$testdir"
+
+cleanup () {
+	rm -rf "$testdir"
+}
+trap cleanup 0
+
+cd "$testdir"
+
+SECRET_BACKEND=file
+export SECRET_BACKEND
+
+SECRET_FILE_TEST_PATH=$testdir/keyring
+export SECRET_FILE_TEST_PATH
+
+SECRET_FILE_TEST_PASSWORD=test
+export SECRET_FILE_TEST_PASSWORD
+
+: ${SECRET_TOOL="$abs_top_builddir"/tool/secret-tool}
+
+: ${DIFF=diff}
+
+echo 1..4
+
+echo test1 | ${SECRET_TOOL} store --label label1 foo bar
+if test $? -eq 0; then
+  echo "ok 1 /secret-tool/store"
+else
+  echo "not ok 1 /secret-tool/store"
+fi
+
+echo test2 | ${SECRET_TOOL} store --label label2 foo bar apple orange
+if test $? -eq 0; then
+  echo "ok 1 /secret-tool/store"
+else
+  echo "not ok 1 /secret-tool/store"
+fi
+
+echo test1 > lookup.exp
+${SECRET_TOOL} lookup foo bar > lookup.out
+if ${DIFF} lookup.exp lookup.out > lookup.diff; then
+  echo "ok 2 /secret-tool/lookup"
+else
+  echo "not ok 2 /secret-tool/lookup"
+  sed 's/^/# /' lookup.diff
+  exit 1
+fi
+
+cat > search.exp <<EOF
+[no path]
+label = label1
+secret = test1
+
+[no path]
+label = label2
+secret = test2
+
+EOF
+
+${SECRET_TOOL} search foo bar | sed '/^created\|^modified/d' > search.out
+if test $? -ne 0; then
+  echo "not ok 3 /secret-tool/search"
+  exit 1
+fi
+if ${DIFF} search.exp search.out > search.diff; then
+  echo "ok 3 /secret-tool/search"
+else
+  echo "not ok 3 /secret-tool/search"
+  sed 's/^/# /' search.diff
+  exit 1
+fi
+
+${SECRET_TOOL} clear apple orange
+if test $? -eq 0; then
+  echo "ok 4 /secret-tool/clear"
+else
+  echo "not ok 4 /secret-tool/clear"
+  exit 1
+fi
+
+cat > search-after-clear.exp <<EOF
+[no path]
+label = label1
+secret = test1
+
+EOF
+
+${SECRET_TOOL} search foo bar | sed '/^created\|^modified/d' > search-after-clear.out
+if test $? -ne 0; then
+  echo "not ok 5 /secret-tool/search-after-clear"
+  exit 1
+fi
+if ${DIFF} search-after-clear.exp search-after-clear.out > search-after-clear.diff; then
+  echo "ok 5 /secret-tool/search-after-clear"
+else
+  echo "not ok 5 /secret-tool/search-after-clear"
+  sed 's/^/# /' search-after-clear.diff
+  exit 1
+fi