os, dix, mi, render: optimize RAM consumption in background when idle

When the X server runs for extended periods or handles bursty client
workloads, significant memory bloat is retained across heap arenas,
unshrunk resource tables, event queues, client I/O buffers, and dormant
shared libraries even after all client resources are destroyed.

Implement comprehensive background idle memory reclamation without
changing memory allocators and preserving full ABI compatibility:

- Startup allocator tuning:
  * Configure M_TRIM_THRESHOLD (128 KB) and M_ARENA_MAX (2).
  * Configure M_MMAP_THRESHOLD (128 KB) to prevent runaway dynamic
    threshold escalation that causes heap fragmentation.

- Idle background trim timer:
  * Schedule an ABI-safe idle callback after 5 seconds of inactivity.
  * Disarm the timer on incoming client traffic so active workloads
    suffer zero overhead or latency penalty.
  * Re-arm periodically (every 60 seconds) during sustained idle states,
    and trigger trim on screensaver / DPMS power-saving events.

- Buffer and cache reclamation:
  * Reclaim idle client input (oc->input) and output (oc->output) buffers,
    along with AvailableInput and cached free buffer pools.
  * Empty dix font pattern caches (patternCache and fontPatternCache).
  * Downsize client resource bucket tables back to INITBUCKETS when
    client resource counts shrink (preserving serverClient index 0).
  * Compact mi input event queue (miEventQueue) back to initial size.
  * Compact global glyph hash tables and clear deleted glyph tombstones.

- Clean non-core library page reclamation:
  * Inspect /proc/self/maps on Linux and issue madvise(MADV_DONTNEED) on
    purely clean file-backed segments (executable r-xp code and file-offset
    zero rodata) of heavy dormant libraries (libLLVM, libz3, libstdc++,
    libxml2, and Mesa DRI drivers).
  * Strictly avoid writable or GNU_RELRO segments to preserve copy-on-write
    dynamic relocations.
  * Invoke malloc_trim(0) to release freed heap pages to the kernel.

- Robust listener socket binding:
  * In _XSERVTransSocketUNIXCreateListener(), attempt direct bind() first
    and only inspect/unlink stale files on EADDRINUSE to prevent race
    conditions between concurrent servers under -displayfd.

- CI compatibility:
  * Guard M_ARENA_MAX and HAVE_MALLOPT for Cygwin.

Signed-off-by: Herman Semenoff <GermanAizek@yandex.ru>

Xext/render/glyph.c

@@ -24,6 +24,7 @@
 
 #include <dix-config.h>
 
+#include "dix/dix_priv.h"
 #include "dix/screenint_priv.h"
 #include "include/mipict.h"
 #include "include/misc.h"
@@ -692,3 +693,15 @@ void SetGlyphPicture(GlyphPtr glyph, ScreenPtr pScreen, PicturePtr picture)
 {
     GlyphPicture(glyph)[pScreen->myNum] = picture;
 }
+
+void
+dixTrimGlyphHash(void)
+{
+    int fdepth;
+
+    for (fdepth = 0; fdepth < GlyphFormatNum; fdepth++) {
+        if (globalGlyphs[fdepth].hashSet && globalGlyphs[fdepth].table) {
+            ResizeGlyphHash(&globalGlyphs[fdepth], 0, TRUE);
+        }
+    }
+}

dix/dix_priv.h

@@ -885,4 +885,20 @@ x_rpcbuf_t dixBuildConnectionBlock(int maxscreens);
  */
 char *dixNewConnectionInfoBlock(ClientPtr pClient, size_t *sz, size_t *scrOffset);
 
+/*
+ * empty the font pattern cache
+ */
+void dixEmptyFontPatternCache(void);
+
+/*
+ * trim resource tables for idle clients
+ */
+void dixTrimResourceTables(void);
+
+/*
+ * trim global glyph hash tables
+ */
+void dixTrimGlyphHash(void);
+
+
 #endif /* _XSERVER_DIX_PRIV_H */

dix/dixfonts.c

@@ -2095,3 +2095,12 @@ InitFonts(void)
     fontPatternCache = xfont2_make_font_pattern_cache();
     xfont2_init(&xfont2_client_funcs);
 }
+
+void
+dixEmptyFontPatternCache(void)
+{
+    if (patternCache)
+        xfont2_empty_font_pattern_cache(patternCache);
+    if (fontPatternCache)
+        xfont2_empty_font_pattern_cache(fontPatternCache);
+}

dix/resource.c

@@ -887,6 +887,60 @@ RebuildTable(int client)
     clientTable[client].resources = resources;
 }
 
+static void
+ShrinkClientResourceTable(int client)
+{
+    int j, new_buckets;
+    ResourcePtr **tails, *resources;
+
+    if (clientTable[client].buckets <= INITBUCKETS ||
+        clientTable[client].hashsize <= INITHASHSIZE)
+        return;
+
+    new_buckets = clientTable[client].buckets / 2;
+    j = new_buckets;
+    tails = calloc(j, sizeof(ResourcePtr *));
+    if (!tails)
+        return;
+    resources = calloc(j, sizeof(ResourcePtr));
+    if (!resources) {
+        free(tails);
+        return;
+    }
+    for (ResourcePtr *rptr = resources, **tptr = tails; --j >= 0; rptr++, tptr++) {
+        *rptr = NULL;
+        *tptr = rptr;
+    }
+    clientTable[client].hashsize--;
+    j = clientTable[client].buckets;
+    for (ResourcePtr *rptr = clientTable[client].resources, **tptr; --j >= 0; rptr++) {
+        for (ResourcePtr res = *rptr, next; res; res = next) {
+            next = res->next;
+            res->next = NULL;
+            tptr = &tails[HashResourceID(res->id, clientTable[client].hashsize)];
+            **tptr = res;
+            *tptr = &res->next;
+        }
+    }
+    free(tails);
+    clientTable[client].buckets = new_buckets;
+    free(clientTable[client].resources);
+    clientTable[client].resources = resources;
+}
+
+void
+dixTrimResourceTables(void)
+{
+    for (int i = 1; i < currentMaxClients; i++) {
+        if (!clientTable[i].resources || !clientTable[i].buckets)
+            continue;
+        while (clientTable[i].buckets > INITBUCKETS &&
+               clientTable[i].elements <= (clientTable[i].buckets / 2)) {
+            ShrinkClientResourceTable(i);
+        }
+    }
+}
+
 static void
 doFreeResource(ResourcePtr res, Bool skip)
 {

include/meson.build

@@ -182,6 +182,12 @@ conf_data.set('HAVE_WALKCONTEXT', cc.has_function('walkcontext') ? '1' : false)
 conf_data.set('HAVE_XUCRED_CR_PID', cc.has_member('struct xucred', 'cr_pid', prefix : '#include <sys/ucred.h>') ? '1' : false)
 conf_data.set('LOCK_SERVER', host_machine.system() != 'windows' ? '1' : false)
 
+if get_option('background_memory_trim')
+    conf_data.set('HAVE_MALLOC_TRIM', cc.has_function('malloc_trim', prefix: '#include <malloc.h>') ? '1' : false)
+    conf_data.set('HAVE_MALLOPT', cc.has_function('mallopt', prefix: '#include <malloc.h>') ? '1' : false)
+    conf_data.set('HAVE_MALLOC_H', cc.has_header('malloc.h') ? '1' : false)
+endif
+
 # Don't let X dependencies typedef 'pointer'
 conf_data.set('_XTYPEDEF_POINTER', '1')
 conf_data.set('_XITYPEDEF_POINTER', '1')

meson_options.txt

@@ -185,3 +185,6 @@ option('test_rendercheck_triangles', type: 'boolean', value: false,
        description: 'testsuite: run rendercheck triangles tests (might fail on Xephyr)')
 option('test_xephyr_gles', type: 'boolean', value: true,
        description: 'testsuite: run gles2/gles3 tests on Xephyr (might fail w/o DRI)')
+
+option('background_memory_trim', type: 'boolean', value: true,
+       description: 'Trim memory heap and buffers when X server is idle in the background')

mi/mi_priv.h

@@ -38,6 +38,7 @@ void miStepDash(int dist, int * pDashIndex, unsigned char * pDash,
 
 Bool mieqInit(void);
 void mieqFini(void);
+void mieqTrimQueue(void);
 void mieqEnqueue(DeviceIntPtr pDev, InternalEvent *e);
 void mieqSwitchScreen(DeviceIntPtr pDev, ScreenPtr pScreen, Bool set_dequeue_screen);
 void mieqProcessDeviceEvent(DeviceIntPtr dev, InternalEvent *event, ScreenPtr screen);

mi/mieq.c

@@ -195,6 +195,33 @@ mieqFini(void)
     free(miEventQueue.events);
 }
 
+void
+mieqTrimQueue(void)
+{
+    input_lock();
+    if (miEventQueue.events && miEventQueue.nevents > QUEUE_INITIAL_SIZE &&
+        mieqNumEnqueued(&miEventQueue) == 0) {
+        EventRec *new_events = calloc(QUEUE_INITIAL_SIZE, sizeof(EventRec));
+        if (new_events) {
+            for (size_t i = 0; i < QUEUE_INITIAL_SIZE; i++) {
+                new_events[i] = miEventQueue.events[i];
+            }
+            for (size_t i = QUEUE_INITIAL_SIZE; i < miEventQueue.nevents; i++) {
+                if (miEventQueue.events[i].events != NULL) {
+                    FreeEventList(miEventQueue.events[i].events, 1);
+                    miEventQueue.events[i].events = NULL;
+                }
+            }
+            free(miEventQueue.events);
+            miEventQueue.events = new_events;
+            miEventQueue.nevents = QUEUE_INITIAL_SIZE;
+            miEventQueue.head = 0;
+            miEventQueue.tail = 0;
+        }
+    }
+    input_unlock();
+}
+
 /*
  * Must be reentrant with ProcessInputEvents.  Assumption: mieqEnqueue
  * will never be interrupted. Must be called with input_lock held

os/WaitFor.c

@@ -54,6 +54,10 @@ SOFTWARE.
 
 #include <dix-config.h>
 
+#ifdef HAVE_MALLOC_H
+#include <malloc.h>
+#endif
+
 #include <stdbool.h>
 #include <errno.h>
 #include <stdio.h>
@@ -67,6 +71,7 @@ SOFTWARE.
 #include "dix/dixstruct_priv.h"
 #include "dix/screensaver_priv.h"
 #include "include/misc.h"
+#include "mi/mi_priv.h"
 #include "os/busfault.h"
 #include "os/client_priv.h"
 #include "os/mathx_priv.h"
@@ -165,6 +170,60 @@ check_timers(void)
  *     pClientsReady is an array to store ready client->index values into.
  *****************/
 
+#ifdef HAVE_MALLOC_TRIM
+#define IDLE_TRIM_DELAY 5000 /* 5 seconds */
+#define IDLE_TRIM_INTERVAL 60000 /* 60 seconds */
+
+static OsTimerPtr IdleTrimTimer = NULL;
+static Bool initial_idle_armed = FALSE;
+
+#if defined(__linux__)
+#include <sys/mman.h>
+#if defined(MADV_DONTNEED)
+
+static void
+OsTrimCleanLibraries(void)
+{
+    FILE *f = fopen("/proc/self/maps", "r");
+    if (!f)
+        return;
+    char line[1024];
+    while (fgets(line, sizeof(line), f)) {
+        unsigned long start, end, offset;
+        char perms[16], path[512] = "";
+        int n = sscanf(line, "%lx-%lx %15s %lx %*x:%*x %*u %511s", &start, &end, perms, &offset, path);
+        if (n >= 5 && path[0] == '/') {
+            Bool is_clean = (strncmp(perms, "r-xp", 4) == 0) ||
+                            (strncmp(perms, "r--p", 4) == 0 && offset == 0);
+            if (is_clean &&
+                (strstr(path, "libLLVM") || strstr(path, "libz3") ||
+                 strstr(path, "libstdc++") || strstr(path, "libxml2") ||
+                 strstr(path, "dri/"))) {
+                madvise((void *) start, end - start, MADV_DONTNEED);
+            }
+        }
+    }
+    fclose(f);
+}
+#endif
+#endif
+
+static CARD32
+IdleTrimCallback(OsTimerPtr timer, CARD32 now, void *arg)
+{
+    ResetOsBuffers();
+    dixEmptyFontPatternCache();
+    dixTrimResourceTables();
+    dixTrimGlyphHash();
+    mieqTrimQueue();
+    malloc_trim(0);
+#if defined(__linux__) && defined(MADV_DONTNEED)
+    OsTrimCleanLibraries();
+#endif
+    return IDLE_TRIM_INTERVAL;
+}
+#endif
+
 Bool
 WaitForSomething(Bool are_ready)
 {
@@ -177,7 +236,18 @@ WaitForSomething(Bool are_ready)
     if (were_ready && !are_ready) {
         timer_is_running = FALSE;
         SmartScheduleStopTimer();
+#ifdef HAVE_MALLOC_TRIM
+        IdleTrimTimer = TimerSet(IdleTrimTimer, 0, IDLE_TRIM_DELAY,
+                                 IdleTrimCallback, NULL);
+#endif
     }
+#ifdef HAVE_MALLOC_TRIM
+    else if (!initial_idle_armed && !are_ready) {
+        initial_idle_armed = TRUE;
+        IdleTrimTimer = TimerSet(IdleTrimTimer, 0, IDLE_TRIM_DELAY,
+                                 IdleTrimCallback, NULL);
+    }
+#endif
 
     were_ready = FALSE;
 
@@ -222,6 +292,10 @@ WaitForSomething(Bool are_ready)
 
         if (are_ready) {
             were_ready = TRUE;
+#ifdef HAVE_MALLOC_TRIM
+            if (IdleTrimTimer)
+                TimerCancel(IdleTrimTimer);
+#endif
             if (!timer_is_running)
                 SmartScheduleStartTimer();
             return TRUE;
@@ -458,6 +532,9 @@ ScreenSaverTimeoutExpire(OsTimerPtr timer, CARD32 now, void *arg)
     }
 
     ResetOsBuffers();           /* not ideal, but better than nothing */
+#ifdef HAVE_MALLOC_TRIM
+    malloc_trim(0);
+#endif
     dixSaveScreens(serverClient, SCREEN_SAVER_ON, ScreenSaverActive);
 
     if (ScreenSaverInterval > 0) {
@@ -477,6 +554,13 @@ FreeScreenSaverTimer(void)
         TimerFree(ScreenSaverTimer);
         ScreenSaverTimer = NULL;
     }
+#ifdef HAVE_MALLOC_TRIM
+    if (IdleTrimTimer) {
+        TimerFree(IdleTrimTimer);
+        IdleTrimTimer = NULL;
+    }
+    initial_idle_armed = FALSE;
+#endif
 }
 
 void

os/Xtranssock.c

@@ -899,20 +899,28 @@ static int _XSERVTransSocketUNIXCreateListener (
 #endif
 
     /*
-     * Don't blindly unlink a pre-existing socket file: it may belong to a
-     * server that's still running (see unix_socket_is_live()). Only clean
-     * up and rebind if nobody answers there.
+     * Try to bind directly without unlinking first. If the socket file does
+     * not exist, bind() will succeed immediately, avoiding races between
+     * concurrent servers.
+     * Only if bind() fails with EADDRINUSE / TRANS_ADDR_IN_USE do we check
+     * whether the socket is actually live; if dead/stale, unlink and retry.
      */
-    if (unix_socket_is_live(sockname.sun_path)) {
-	prmsg (1, "SocketUNIXCreateListener: %s already in use\n",
-	       sockname.sun_path);
-	(void) umask (oldUmask);
-	return TRANS_ADDR_IN_USE;
+    status = _XSERVTransSocketCreateListener (ciptr,
+	(struct sockaddr *) &sockname, namelen, flags);
+    if (status < 0) {
+	if (status == TRANS_ADDR_IN_USE || errno == EADDRINUSE) {
+	    if (unix_socket_is_live(sockname.sun_path)) {
+		prmsg (1, "SocketUNIXCreateListener: %s already in use\n",
+		       sockname.sun_path);
+		(void) umask (oldUmask);
+		return TRANS_ADDR_IN_USE;
+	    }
+	    unlink (sockname.sun_path);
+	    status = _XSERVTransSocketCreateListener (ciptr,
+		(struct sockaddr *) &sockname, namelen, flags);
+	}
     }
-    unlink (sockname.sun_path);
-
-    if ((status = _XSERVTransSocketCreateListener (ciptr,
-	(struct sockaddr *) &sockname, namelen, flags)) < 0)
+    if (status < 0)
     {
 	prmsg (1,
     "SocketUNIXCreateListener: ...SocketCreateListener() failed\n");

os/io.c

@@ -1110,6 +1110,37 @@ ResetOsBuffers(void)
 {
     ConnectionInputPtr oci;
     ConnectionOutputPtr oco;
+    int i;
+
+    if (AvailableInput) {
+        ConnectionInputPtr aci = AvailableInput->input;
+        if (aci) {
+            free(aci->buffer);
+            free(aci);
+            AvailableInput->input = NULL;
+        }
+        AvailableInput = NULL;
+    }
+
+    for (i = 1; i < currentMaxClients; i++) {
+        ClientPtr client = clients[i];
+        if (client && !client->clientGone && client->osPrivate) {
+            OsCommPtr oc = (OsCommPtr) client->osPrivate;
+            if ((oci = oc->input)) {
+                int gotnow = oci->bufcnt + oci->buffer - (oci->bufptr + oci->lenLastReq);
+                if (gotnow <= 0 && oci->ignoreBytes == 0) {
+                    free(oci->buffer);
+                    free(oci);
+                    oc->input = NULL;
+                }
+            }
+            if ((oco = oc->output) && oco->count == 0) {
+                free(oco->buf);
+                free(oco);
+                oc->output = NULL;
+            }
+        }
+    }
 
     while ((oci = FreeInputs)) {
         FreeInputs = oci->next;

os/osinit.c

@@ -46,6 +46,10 @@ SOFTWARE.
 
 #include <dix-config.h>
 
+#ifdef HAVE_MALLOC_H
+#include <malloc.h>
+#endif
+
 #include <errno.h>
 #include <stdio.h>
 #include <signal.h>
@@ -182,6 +186,18 @@ OsInit(void)
         }
 #endif /* !WIN32 || __CYGWIN__ */
         busfault_init();
+#if defined(HAVE_MALLOC_H) && defined(HAVE_MALLOC_TRIM) && defined(HAVE_MALLOPT)
+        /* Prevent runaway dynamic mmap/trim thresholds and limit arena fragmentation */
+#ifdef M_TRIM_THRESHOLD
+        mallopt(M_TRIM_THRESHOLD, 128 * 1024);
+#endif
+#ifdef M_MMAP_THRESHOLD
+        mallopt(M_MMAP_THRESHOLD, 128 * 1024);
+#endif
+#ifdef M_ARENA_MAX
+        mallopt(M_ARENA_MAX, 2);
+#endif
+#endif
         server_poll = ospoll_create();
         if (!server_poll)
             FatalError("failed to allocate poll structure");