diff --git a/libpkgconf/cache.c b/libpkgconf/cache.c
index 2630855..6b9ca8a 100644
--- a/libpkgconf/cache.c
+++ b/libpkgconf/cache.c
@@ -233,7 +233,7 @@ pkgconf_cache_free(pkgconf_client_t *client)
 	for (i = 0, count = client->cache_count; i < count; i++)
 	{
 		pkgconf_pkg_t *pkg = cache_table[i];
-		pkgconf_pkg_free(client, pkg);
+		pkgconf_pkg_unref(client, pkg);
 	}
 	free(cache_table);
 
diff --git a/libpkgconf/pkg.c b/libpkgconf/pkg.c
index 577f0ad..ed3fd5f 100644
--- a/libpkgconf/pkg.c
+++ b/libpkgconf/pkg.c
@@ -17,6 +17,8 @@
 #include <libpkgconf/stdinc.h>
 #include <libpkgconf/libpkgconf.h>
 
+#include <assert.h>
+
 /*
  * !doc
  *
@@ -514,6 +516,8 @@ pkgconf_pkg_free(pkgconf_client_t *client, pkgconf_pkg_t *pkg)
 	if (pkg->pc_filedir != NULL)
 		free(pkg->pc_filedir);
 
+	/* We shouldn't be trying to free this if someone is still holding a reference to it */
+	assert(pkg->refcount == 0);
 	free(pkg);
 }