...
This commit is contained in:
@@ -40,6 +40,8 @@ import java.nio.file.Paths;
|
||||
import java.nio.file.StandardOpenOption;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.Semaphore;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
@@ -50,13 +52,26 @@ public class PlineTools {
|
||||
|
||||
private static final int DEFAULT_AMI_COMMAND_CONNECTIONS = 4;
|
||||
private static final long AMI_COMMAND_TIMEOUT_MS = 5000;
|
||||
private static final long AMI_RELOAD_TIMEOUT_MS = 60_000;
|
||||
private static final long DEFAULT_AMI_COMMAND_IDLE_TIMEOUT_MS = 30_000;
|
||||
private static final long AMI_COMMAND_POOL_SWEEP_INTERVAL_SECONDS = 10;
|
||||
private static final long ASTERISK_CLI_TIMEOUT_SECONDS = 10;
|
||||
private static final long ASTERISK_RELOAD_CLI_TIMEOUT_SECONDS = 60;
|
||||
private final Object amiCommandPoolLock = new Object();
|
||||
private final AtomicInteger amiCommandConnectionsCreated = new AtomicInteger();
|
||||
private final ArrayBlockingQueue<ManagerConnection> idleAmiCommandConnections =
|
||||
private final ArrayBlockingQueue<IdleAmiCommandConnection> idleAmiCommandConnections =
|
||||
new ArrayBlockingQueue<>(DEFAULT_AMI_COMMAND_CONNECTIONS);
|
||||
private final Semaphore amiCommandSlots = new Semaphore(DEFAULT_AMI_COMMAND_CONNECTIONS, true);
|
||||
private final Semaphore localCliFallbackSlots = new Semaphore(2, true);
|
||||
private final ScheduledExecutorService amiCommandPoolMaintenance =
|
||||
Executors.newSingleThreadScheduledExecutor(runnable -> {
|
||||
Thread thread = new Thread(runnable, "ami-command-pool-maintenance");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
|
||||
@Value("${pline.ami-command-idle-timeout-ms:" + DEFAULT_AMI_COMMAND_IDLE_TIMEOUT_MS + "}")
|
||||
private long amiCommandIdleTimeoutMs = DEFAULT_AMI_COMMAND_IDLE_TIMEOUT_MS;
|
||||
|
||||
private AppSettingsService appSettingsService;
|
||||
|
||||
@@ -281,6 +296,11 @@ public class PlineTools {
|
||||
|
||||
this.logger = LogManager.getLogger(PlineTools.class);
|
||||
ConnectSocketToCore();
|
||||
amiCommandPoolMaintenance.scheduleWithFixedDelay(
|
||||
this::expireIdleAmiCommandConnections,
|
||||
AMI_COMMAND_POOL_SWEEP_INTERVAL_SECONDS,
|
||||
AMI_COMMAND_POOL_SWEEP_INTERVAL_SECONDS,
|
||||
TimeUnit.SECONDS);
|
||||
tmrTryConnect = new Timer();
|
||||
tmrTryConnect.scheduleAtFixedRate(new TimerTask() {
|
||||
@Override
|
||||
@@ -429,9 +449,11 @@ public class PlineTools {
|
||||
return wantResult ? String.join("\n", result) + (result.isEmpty() ? "" : "\n") : "";
|
||||
}
|
||||
|
||||
private record IdleAmiCommandConnection(ManagerConnection connection, long returnedAtNanos) { }
|
||||
|
||||
private record AmiCommandAttempt(boolean actionSent, List<String> output) { }
|
||||
|
||||
private AmiCommandAttempt executeAmiCommand(String command) {
|
||||
private AmiCommandAttempt executeAmiCommand(String command, long timeoutMs) {
|
||||
if (!amiCommandSlots.tryAcquire()) {
|
||||
plineLogger("AMI command pool is busy; using local Asterisk CLI fallback");
|
||||
return new AmiCommandAttempt(false, null);
|
||||
@@ -451,7 +473,7 @@ public class PlineTools {
|
||||
// Once sendAction starts, a timeout has an uncertain outcome. Never retry it
|
||||
// through the CLI, since the first execution might already have completed.
|
||||
actionSent = true;
|
||||
var response = connection.sendAction(new CommandAction(command), AMI_COMMAND_TIMEOUT_MS);
|
||||
var response = connection.sendAction(new CommandAction(command), timeoutMs);
|
||||
reusable = connection.getState() == ManagerConnectionState.CONNECTED;
|
||||
if (response instanceof CommandResponse commandResponse)
|
||||
return new AmiCommandAttempt(true, commandResponse.getResult());
|
||||
@@ -464,7 +486,8 @@ public class PlineTools {
|
||||
} finally {
|
||||
if (connection != null) {
|
||||
if (reusable && connection.getState() == ManagerConnectionState.CONNECTED
|
||||
&& idleAmiCommandConnections.offer(connection)) {
|
||||
&& idleAmiCommandConnections.offer(
|
||||
new IdleAmiCommandConnection(connection, System.nanoTime()))) {
|
||||
// Keep healthy connections warm for the next command.
|
||||
} else {
|
||||
discardAmiCommandConnection(connection);
|
||||
@@ -475,9 +498,16 @@ public class PlineTools {
|
||||
}
|
||||
|
||||
private ManagerConnection borrowAmiCommandConnection() {
|
||||
ManagerConnection idle = idleAmiCommandConnections.poll();
|
||||
if (idle != null)
|
||||
return idle;
|
||||
IdleAmiCommandConnection idle;
|
||||
while ((idle = idleAmiCommandConnections.poll()) != null) {
|
||||
ManagerConnection connection = idle.connection();
|
||||
if (isAmiCommandConnectionExpired(idle)
|
||||
|| connection.getState() != ManagerConnectionState.CONNECTED) {
|
||||
discardAmiCommandConnection(connection);
|
||||
continue;
|
||||
}
|
||||
return connection;
|
||||
}
|
||||
|
||||
synchronized (amiCommandPoolLock) {
|
||||
if (amiCommandConnectionsCreated.get() >= DEFAULT_AMI_COMMAND_CONNECTIONS)
|
||||
@@ -507,6 +537,31 @@ public class PlineTools {
|
||||
}
|
||||
}
|
||||
|
||||
private boolean isAmiCommandConnectionExpired(IdleAmiCommandConnection idle) {
|
||||
long idleTimeoutMs = Math.max(1, amiCommandIdleTimeoutMs);
|
||||
long idleNanos = System.nanoTime() - idle.returnedAtNanos();
|
||||
return idleNanos >= TimeUnit.MILLISECONDS.toNanos(idleTimeoutMs);
|
||||
}
|
||||
|
||||
private void expireIdleAmiCommandConnections() {
|
||||
try {
|
||||
List<IdleAmiCommandConnection> keep = new ArrayList<>();
|
||||
IdleAmiCommandConnection idle;
|
||||
while ((idle = idleAmiCommandConnections.poll()) != null) {
|
||||
if (isAmiCommandConnectionExpired(idle))
|
||||
discardAmiCommandConnection(idle.connection());
|
||||
else
|
||||
keep.add(idle);
|
||||
}
|
||||
for (IdleAmiCommandConnection connection : keep) {
|
||||
if (!idleAmiCommandConnections.offer(connection))
|
||||
discardAmiCommandConnection(connection.connection());
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
plineLogger("AMI command pool idle cleanup failed: " + ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
private void discardAmiCommandConnection(ManagerConnection connection) {
|
||||
try {
|
||||
if (connection.getState() == ManagerConnectionState.CONNECTED)
|
||||
@@ -520,26 +575,35 @@ public class PlineTools {
|
||||
|
||||
@PreDestroy
|
||||
public void closeAmiCommandConnections() {
|
||||
ManagerConnection connection;
|
||||
while ((connection = idleAmiCommandConnections.poll()) != null)
|
||||
discardAmiCommandConnection(connection);
|
||||
amiCommandPoolMaintenance.shutdownNow();
|
||||
IdleAmiCommandConnection idle;
|
||||
while ((idle = idleAmiCommandConnections.poll()) != null)
|
||||
discardAmiCommandConnection(idle.connection());
|
||||
}
|
||||
|
||||
public record AsteriskCliExecution(List<String> output, String transport) { }
|
||||
|
||||
public AsteriskCliExecution executeAsteriskCliCommand(String command) {
|
||||
return executeAsteriskCliCommand(command, AMI_COMMAND_TIMEOUT_MS, ASTERISK_CLI_TIMEOUT_SECONDS);
|
||||
}
|
||||
|
||||
private AsteriskCliExecution executeAsteriskCliCommand(String command, long amiTimeoutMs,
|
||||
long cliTimeoutSeconds) {
|
||||
if (command == null || command.isBlank() || command.length() > 4096
|
||||
|| command.indexOf('\n') >= 0 || command.indexOf('\r') >= 0 || command.indexOf('\0') >= 0) {
|
||||
plineLogger("Rejected invalid Asterisk CLI command input");
|
||||
return new AsteriskCliExecution(null, "NONE");
|
||||
}
|
||||
AmiCommandAttempt amiAttempt = executeAmiCommand(command);
|
||||
AmiCommandAttempt amiAttempt = executeAmiCommand(command, amiTimeoutMs);
|
||||
if (amiAttempt.actionSent()) {
|
||||
plineLogger("Asterisk CLI command completed through AMI");
|
||||
if (amiAttempt.output() == null)
|
||||
plineLogger("AMI command was sent but its result is unknown");
|
||||
else
|
||||
plineLogger("Asterisk CLI command completed through AMI");
|
||||
return new AsteriskCliExecution(amiAttempt.output(), "AMI");
|
||||
}
|
||||
|
||||
String cliOutput = executeAsteriskCliFallback(command);
|
||||
String cliOutput = executeAsteriskCliFallback(command, cliTimeoutSeconds);
|
||||
if (cliOutput == null)
|
||||
return new AsteriskCliExecution(null, "CLI");
|
||||
List<String> lines = cliOutput.isEmpty() ? List.of() : cliOutput.lines().toList();
|
||||
@@ -551,7 +615,7 @@ public class PlineTools {
|
||||
return executeAsteriskCliCommand(command).output();
|
||||
}
|
||||
|
||||
private String executeAsteriskCliFallback(String command) {
|
||||
private String executeAsteriskCliFallback(String command, long timeoutSeconds) {
|
||||
if (!localCliFallbackSlots.tryAcquire()) {
|
||||
plineLogger("Local Asterisk CLI fallback is busy; rejecting this execution without queueing");
|
||||
return null;
|
||||
@@ -573,10 +637,10 @@ public class PlineTools {
|
||||
ProcessBuilder pb = new ProcessBuilder(cmd).redirectErrorStream(true)
|
||||
.redirectOutput(outputFile.toFile());
|
||||
process = pb.start();
|
||||
if (!process.waitFor(ASTERISK_CLI_TIMEOUT_SECONDS, TimeUnit.SECONDS)) {
|
||||
if (!process.waitFor(timeoutSeconds, TimeUnit.SECONDS)) {
|
||||
process.destroyForcibly();
|
||||
process.waitFor();
|
||||
plineLogger("Asterisk CLI fallback timed out after " + ASTERISK_CLI_TIMEOUT_SECONDS + " seconds");
|
||||
plineLogger("Asterisk CLI fallback timed out after " + timeoutSeconds + " seconds");
|
||||
return null;
|
||||
}
|
||||
if (process.exitValue() != 0) {
|
||||
@@ -648,7 +712,7 @@ public class PlineTools {
|
||||
return result == null ? null : String.join("\n", result) + (result.isEmpty() ? "" : "\n");
|
||||
}
|
||||
|
||||
public void reloadConfigurations() {
|
||||
public boolean reloadConfigurations() {
|
||||
// Apply rewrites Asterisk configuration only. Recursively changing the
|
||||
// owner of the spool/lib trees walks recordings and voicemail data and
|
||||
// can make a routine Apply take minutes on a live PBX.
|
||||
@@ -658,10 +722,23 @@ public class PlineTools {
|
||||
this.getConfigPath()
|
||||
}, true);
|
||||
|
||||
// Apply manager.conf changes (including allowmultiplelogin) before the
|
||||
// command connections are reused; this is also needed on in-place upgrades.
|
||||
execCoreCommand("manager reload");
|
||||
execCoreCommand("module reload");
|
||||
// The global module reload below also applies manager.conf changes,
|
||||
// including allowmultiplelogin, during in-place upgrades.
|
||||
// A full module reload also reloads the manager module. Send one reload
|
||||
// request, allow it longer than ordinary CLI commands, and surface an
|
||||
// in-progress reload instead of reporting Apply as successful.
|
||||
AsteriskCliExecution execution = executeAsteriskCliCommand("module reload", AMI_RELOAD_TIMEOUT_MS,
|
||||
ASTERISK_RELOAD_CLI_TIMEOUT_SECONDS);
|
||||
if (execution.output() == null) {
|
||||
plineLogger("Asterisk reload did not return a result; it may still be running. Check Asterisk logs.");
|
||||
return false;
|
||||
}
|
||||
String output = String.join("\n", execution.output());
|
||||
if (output.toLowerCase(Locale.ROOT).contains("the previous reload command didn't finish yet")) {
|
||||
plineLogger("Asterisk rejected the reload because a previous reload is still running.");
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
public boolean checkSipUid(String uid) {
|
||||
|
||||
Reference in New Issue
Block a user