This commit is contained in:
2026-10-20 11:49:12 +03:30
parent 3f72861b64
commit 10978d47a5
35 changed files with 683 additions and 644 deletions

View File

@@ -40,6 +40,8 @@ import java.nio.file.Paths;
import java.nio.file.StandardOpenOption;
import java.util.*;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.Semaphore;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@@ -50,13 +52,26 @@ public class PlineTools {
private static final int DEFAULT_AMI_COMMAND_CONNECTIONS = 4;
private static final long AMI_COMMAND_TIMEOUT_MS = 5000;
private static final long AMI_RELOAD_TIMEOUT_MS = 60_000;
private static final long DEFAULT_AMI_COMMAND_IDLE_TIMEOUT_MS = 30_000;
private static final long AMI_COMMAND_POOL_SWEEP_INTERVAL_SECONDS = 10;
private static final long ASTERISK_CLI_TIMEOUT_SECONDS = 10;
private static final long ASTERISK_RELOAD_CLI_TIMEOUT_SECONDS = 60;
private final Object amiCommandPoolLock = new Object();
private final AtomicInteger amiCommandConnectionsCreated = new AtomicInteger();
private final ArrayBlockingQueue<ManagerConnection> idleAmiCommandConnections =
private final ArrayBlockingQueue<IdleAmiCommandConnection> idleAmiCommandConnections =
new ArrayBlockingQueue<>(DEFAULT_AMI_COMMAND_CONNECTIONS);
private final Semaphore amiCommandSlots = new Semaphore(DEFAULT_AMI_COMMAND_CONNECTIONS, true);
private final Semaphore localCliFallbackSlots = new Semaphore(2, true);
private final ScheduledExecutorService amiCommandPoolMaintenance =
Executors.newSingleThreadScheduledExecutor(runnable -> {
Thread thread = new Thread(runnable, "ami-command-pool-maintenance");
thread.setDaemon(true);
return thread;
});
@Value("${pline.ami-command-idle-timeout-ms:" + DEFAULT_AMI_COMMAND_IDLE_TIMEOUT_MS + "}")
private long amiCommandIdleTimeoutMs = DEFAULT_AMI_COMMAND_IDLE_TIMEOUT_MS;
private AppSettingsService appSettingsService;
@@ -281,6 +296,11 @@ public class PlineTools {
this.logger = LogManager.getLogger(PlineTools.class);
ConnectSocketToCore();
amiCommandPoolMaintenance.scheduleWithFixedDelay(
this::expireIdleAmiCommandConnections,
AMI_COMMAND_POOL_SWEEP_INTERVAL_SECONDS,
AMI_COMMAND_POOL_SWEEP_INTERVAL_SECONDS,
TimeUnit.SECONDS);
tmrTryConnect = new Timer();
tmrTryConnect.scheduleAtFixedRate(new TimerTask() {
@Override
@@ -429,9 +449,11 @@ public class PlineTools {
return wantResult ? String.join("\n", result) + (result.isEmpty() ? "" : "\n") : "";
}
private record IdleAmiCommandConnection(ManagerConnection connection, long returnedAtNanos) { }
private record AmiCommandAttempt(boolean actionSent, List<String> output) { }
private AmiCommandAttempt executeAmiCommand(String command) {
private AmiCommandAttempt executeAmiCommand(String command, long timeoutMs) {
if (!amiCommandSlots.tryAcquire()) {
plineLogger("AMI command pool is busy; using local Asterisk CLI fallback");
return new AmiCommandAttempt(false, null);
@@ -451,7 +473,7 @@ public class PlineTools {
// Once sendAction starts, a timeout has an uncertain outcome. Never retry it
// through the CLI, since the first execution might already have completed.
actionSent = true;
var response = connection.sendAction(new CommandAction(command), AMI_COMMAND_TIMEOUT_MS);
var response = connection.sendAction(new CommandAction(command), timeoutMs);
reusable = connection.getState() == ManagerConnectionState.CONNECTED;
if (response instanceof CommandResponse commandResponse)
return new AmiCommandAttempt(true, commandResponse.getResult());
@@ -464,7 +486,8 @@ public class PlineTools {
} finally {
if (connection != null) {
if (reusable && connection.getState() == ManagerConnectionState.CONNECTED
&& idleAmiCommandConnections.offer(connection)) {
&& idleAmiCommandConnections.offer(
new IdleAmiCommandConnection(connection, System.nanoTime()))) {
// Keep healthy connections warm for the next command.
} else {
discardAmiCommandConnection(connection);
@@ -475,9 +498,16 @@ public class PlineTools {
}
private ManagerConnection borrowAmiCommandConnection() {
ManagerConnection idle = idleAmiCommandConnections.poll();
if (idle != null)
return idle;
IdleAmiCommandConnection idle;
while ((idle = idleAmiCommandConnections.poll()) != null) {
ManagerConnection connection = idle.connection();
if (isAmiCommandConnectionExpired(idle)
|| connection.getState() != ManagerConnectionState.CONNECTED) {
discardAmiCommandConnection(connection);
continue;
}
return connection;
}
synchronized (amiCommandPoolLock) {
if (amiCommandConnectionsCreated.get() >= DEFAULT_AMI_COMMAND_CONNECTIONS)
@@ -507,6 +537,31 @@ public class PlineTools {
}
}
private boolean isAmiCommandConnectionExpired(IdleAmiCommandConnection idle) {
long idleTimeoutMs = Math.max(1, amiCommandIdleTimeoutMs);
long idleNanos = System.nanoTime() - idle.returnedAtNanos();
return idleNanos >= TimeUnit.MILLISECONDS.toNanos(idleTimeoutMs);
}
private void expireIdleAmiCommandConnections() {
try {
List<IdleAmiCommandConnection> keep = new ArrayList<>();
IdleAmiCommandConnection idle;
while ((idle = idleAmiCommandConnections.poll()) != null) {
if (isAmiCommandConnectionExpired(idle))
discardAmiCommandConnection(idle.connection());
else
keep.add(idle);
}
for (IdleAmiCommandConnection connection : keep) {
if (!idleAmiCommandConnections.offer(connection))
discardAmiCommandConnection(connection.connection());
}
} catch (Exception ex) {
plineLogger("AMI command pool idle cleanup failed: " + ex.getMessage());
}
}
private void discardAmiCommandConnection(ManagerConnection connection) {
try {
if (connection.getState() == ManagerConnectionState.CONNECTED)
@@ -520,26 +575,35 @@ public class PlineTools {
@PreDestroy
public void closeAmiCommandConnections() {
ManagerConnection connection;
while ((connection = idleAmiCommandConnections.poll()) != null)
discardAmiCommandConnection(connection);
amiCommandPoolMaintenance.shutdownNow();
IdleAmiCommandConnection idle;
while ((idle = idleAmiCommandConnections.poll()) != null)
discardAmiCommandConnection(idle.connection());
}
public record AsteriskCliExecution(List<String> output, String transport) { }
public AsteriskCliExecution executeAsteriskCliCommand(String command) {
return executeAsteriskCliCommand(command, AMI_COMMAND_TIMEOUT_MS, ASTERISK_CLI_TIMEOUT_SECONDS);
}
private AsteriskCliExecution executeAsteriskCliCommand(String command, long amiTimeoutMs,
long cliTimeoutSeconds) {
if (command == null || command.isBlank() || command.length() > 4096
|| command.indexOf('\n') >= 0 || command.indexOf('\r') >= 0 || command.indexOf('\0') >= 0) {
plineLogger("Rejected invalid Asterisk CLI command input");
return new AsteriskCliExecution(null, "NONE");
}
AmiCommandAttempt amiAttempt = executeAmiCommand(command);
AmiCommandAttempt amiAttempt = executeAmiCommand(command, amiTimeoutMs);
if (amiAttempt.actionSent()) {
plineLogger("Asterisk CLI command completed through AMI");
if (amiAttempt.output() == null)
plineLogger("AMI command was sent but its result is unknown");
else
plineLogger("Asterisk CLI command completed through AMI");
return new AsteriskCliExecution(amiAttempt.output(), "AMI");
}
String cliOutput = executeAsteriskCliFallback(command);
String cliOutput = executeAsteriskCliFallback(command, cliTimeoutSeconds);
if (cliOutput == null)
return new AsteriskCliExecution(null, "CLI");
List<String> lines = cliOutput.isEmpty() ? List.of() : cliOutput.lines().toList();
@@ -551,7 +615,7 @@ public class PlineTools {
return executeAsteriskCliCommand(command).output();
}
private String executeAsteriskCliFallback(String command) {
private String executeAsteriskCliFallback(String command, long timeoutSeconds) {
if (!localCliFallbackSlots.tryAcquire()) {
plineLogger("Local Asterisk CLI fallback is busy; rejecting this execution without queueing");
return null;
@@ -573,10 +637,10 @@ public class PlineTools {
ProcessBuilder pb = new ProcessBuilder(cmd).redirectErrorStream(true)
.redirectOutput(outputFile.toFile());
process = pb.start();
if (!process.waitFor(ASTERISK_CLI_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
if (!process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) {
process.destroyForcibly();
process.waitFor();
plineLogger("Asterisk CLI fallback timed out after " + ASTERISK_CLI_TIMEOUT_SECONDS + " seconds");
plineLogger("Asterisk CLI fallback timed out after " + timeoutSeconds + " seconds");
return null;
}
if (process.exitValue() != 0) {
@@ -648,7 +712,7 @@ public class PlineTools {
return result == null ? null : String.join("\n", result) + (result.isEmpty() ? "" : "\n");
}
public void reloadConfigurations() {
public boolean reloadConfigurations() {
// Apply rewrites Asterisk configuration only. Recursively changing the
// owner of the spool/lib trees walks recordings and voicemail data and
// can make a routine Apply take minutes on a live PBX.
@@ -658,10 +722,23 @@ public class PlineTools {
this.getConfigPath()
}, true);
// Apply manager.conf changes (including allowmultiplelogin) before the
// command connections are reused; this is also needed on in-place upgrades.
execCoreCommand("manager reload");
execCoreCommand("module reload");
// The global module reload below also applies manager.conf changes,
// including allowmultiplelogin, during in-place upgrades.
// A full module reload also reloads the manager module. Send one reload
// request, allow it longer than ordinary CLI commands, and surface an
// in-progress reload instead of reporting Apply as successful.
AsteriskCliExecution execution = executeAsteriskCliCommand("module reload", AMI_RELOAD_TIMEOUT_MS,
ASTERISK_RELOAD_CLI_TIMEOUT_SECONDS);
if (execution.output() == null) {
plineLogger("Asterisk reload did not return a result; it may still be running. Check Asterisk logs.");
return false;
}
String output = String.join("\n", execution.output());
if (output.toLowerCase(Locale.ROOT).contains("the previous reload command didn't finish yet")) {
plineLogger("Asterisk rejected the reload because a previous reload is still running.");
return false;
}
return true;
}
public boolean checkSipUid(String uid) {